1.14.2

* Update zigbee-herdsman-converters to 12.0.147

* Update homeassistant.js

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2mqtt version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2mqtt fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2mqtt version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.github/stale.yml

@@ -1,5 +1,5 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
+daysUntilStale: 30
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
 # Label to use when marking an issue as stale

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee-herdsman" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters

.gitignore

@@ -67,6 +67,7 @@ data-backup/
 data/coordinator_backup.json
 data/.storage
 data/database.db.backup
+data/extension
 
 # MacOS indexing file
 .DS_Store

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.yaml": "home-assistant"
+    }
+}

README.md

@@ -66,6 +66,9 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

azure-pipelines.yaml

@@ -76,15 +76,13 @@ steps:
   displayName: 'Docker build release'
   condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
 
-# Trigger downstream
+# Trigger Home Assistant Zigbee2mqtt addon build
 - bash: |
-    BODY="{\"request\": {\"branch\": \"master\"}}"
-    curl -s -X POST \
-        -H "Content-Type: application/json" \
-        -H "Accept: application/json" \
-        -H "Travis-API-Version: 3" \
-        -H "Authorization: token $(TRAVIS_TOKEN)" \
-        -d "$BODY" \
-        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
   displayName: 'Trigger Hass.io build'
   condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -3,12 +3,12 @@ FROM node:12-alpine as base
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev
 
-COPY package.json .
+COPY package.json ./
 
 # Dependencies
 FROM base as dependencies
 
-COPY npm-shrinkwrap.json .
+COPY npm-shrinkwrap.json ./
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
     npm ci --production && \
@@ -19,12 +19,15 @@ FROM base as release
 
 COPY --from=dependencies /app/node_modules ./node_modules
 COPY lib ./lib
-COPY index.js docker/run.sh data/configuration.yaml ./
+COPY index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
-RUN chmod +x /app/run.sh
 RUN mkdir /app/data
 
 ARG COMMIT
 RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
 
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["npm", "start"]

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,4 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-exec npm start
+exec "$@"

lib/controller.js

@@ -6,72 +6,87 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const fs = require('fs');
+const data = require('./util/data');
+const path = require('path');
 
 // Extensions
-const ExtensionEntityPublish = require('./extension/entityPublish');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionDeviceEvent = require('./extension/deviceEvent');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/report');
+const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
 
 class Controller {
     constructor() {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
         this.eventBus = new EventBus();
-        this.state = new State();
+        this.state = new State(this.eventBus);
 
         this.publishEntityState = this.publishEntityState.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionEntityPublish(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceGroupMembership(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionDeviceEvent(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
-            new ExtensionOTAUpdate(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
         ];
 
+        if (settings.get().experimental.new_api) {
+            this.extensions.push(new ExtensionBridge(...args));
+        }
+
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionReport(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
         /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
-            ));
+            this.extensions.push(new ExtensionAvailability(...args));
+        }
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
+        }
+
+        const extensionPath = data.joinPath('extension');
+        if (fs.existsSync(extensionPath)) {
+            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
+            for (const extension of extensions) {
+                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
+                this.extensions.push(new Extension(...args, settings, logger));
+            }
         }
     }
 
@@ -114,8 +129,8 @@ class Controller {
             logger.info(
                 (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
                 ` (${entity.device.ieeeAddr}): ` +
-                (entity.mapped ?
-                    `${entity.mapped.model} - ${entity.mapped.vendor} ${entity.mapped.description} ` :
+                (entity.definition ?
+                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
                     'Not supported ') +
                 `(${entity.device.type})`,
             );
@@ -128,14 +143,18 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        await this.zigbee.permitJoin(settings.get().permit_join);
+        try {
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
 
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
         await this.mqtt.connect();
 
         // Send all cached states.
-        if (settings.get().advanced.cache_state) {
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
             for (const device of this.zigbee.getClients()) {
                 if (this.state.exists(device.ieeeAddr)) {
                     this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
@@ -143,6 +162,13 @@ class Controller {
             }
         }
 
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
+
         // Call extensions
         await this.callExtensionMethod('onMQTTConnected', []);
     }
@@ -170,13 +196,13 @@ class Controller {
     }
 
     async onZigbeeEvent(type, data) {
-        const entity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            entity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
         }
 
-        const name = entity && entity.settings ? entity.settings.friendlyName : null;
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
 
         if (type === 'message') {
             logger.debug(
@@ -186,52 +212,38 @@ class Controller {
             );
         } else if (type === 'deviceJoined') {
             logger.info(`Device '${name}' joined`);
-            this.mqtt.log('device_connected', {friendly_name: name});
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
                 logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
 
-                if (entity.mapped) {
-                    const {vendor, description, model} = entity.mapped;
-                    logger.info(
-                        `Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`,
-                    );
-
-                    const log = {friendly_name: name, model, vendor, description, supported: true};
-                    this.mqtt.log('pairing', 'interview_successful', log);
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
                         `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
-                    this.mqtt.log('pairing', 'interview_successful', {friendly_name: name, supported: false});
                 }
             } else if (data.status === 'failed') {
                 logger.error(`Failed to interview '${name}', device has not successfully been paired`);
-                this.mqtt.log('pairing', 'interview_failed', {friendly_name: name});
             } else {
                 /* istanbul ignore else */
                 if (data.status === 'started') {
                     logger.info(`Starting interview of '${name}'`);
-                    this.mqtt.log('pairing', 'interview_started', {friendly_name: name});
                 }
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
-            this.mqtt.log('device_announced', 'announce', {friendly_name: name});
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
                 logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
-                this.mqtt.log('device_removed', 'left_network', {friendly_name: name || data.ieeeAddr});
             }
         }
 
         // Call extensions
-        this.callExtensionMethod(
-            'onZigbeeEvent',
-            [type, data, entity ? entity.mapped : null, entity ? entity.settings : null],
-        );
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
     }
 
     onMQTTMessage(payload) {
@@ -243,22 +255,19 @@ class Controller {
     }
 
     async publishEntityState(IDorName, payload, stateChangeReason=null) {
-        const entity = this.zigbee.resolveEntity(IDorName);
-        if (!entity || !entity.settings) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
             logger.error(`'${IDorName}' does not exist, skipping publish`);
             return;
         }
 
-        if (entity.type === 'device' && settings.get().advanced.last_seen !== 'disable' && entity.device.lastSeen) {
-            payload.last_seen = utils.formatDate(entity.device.lastSeen, settings.get().advanced.last_seen);
-        }
-
         let messagePayload = {...payload};
-        const currentState = this.state.exists(entity.settings.ID) ? this.state.get(entity.settings.ID) : {};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
         const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(entity.settings.ID, newState, stateChangeReason);
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
@@ -267,53 +276,58 @@ class Controller {
 
         const deviceOptions = settings.get().device_options;
         const options = {
-            retain: utils.getObjectsProperty([entity.settings, deviceOptions], 'retain', false),
-            qos: utils.getObjectsProperty([entity.settings, deviceOptions], 'qos', 0),
+            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
+            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
         };
 
-        const retention = utils.getObjectsProperty([entity.settings, deviceOptions], 'retention', false);
+        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention};
         }
 
-        if (entity.type === 'device' && settings.get().mqtt.include_device_information) {
-            const device = this.zigbee.getDeviceByIeeeAddr(entity.device.ieeeAddr);
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
             const attributes = [
                 'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
                 'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
             ];
 
             messagePayload.device = {
-                friendlyName: entity.name,
-                model: entity.mapped ? entity.mapped.model : 'unknown',
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
             };
 
-            attributes.forEach((a) => messagePayload.device[a] = device[a]);
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
         }
 
         // filter mqtt message attributes
         if (deviceOptions.filtered_attributes) {
             deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
-        if (entity.settings.filtered_attributes) {
-            entity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity});
-
         if (Object.entries(messagePayload).length) {
             if (settings.get().experimental.output === 'attribute_and_json') {
-                await this.mqtt.publish(entity.name, JSON.stringify(messagePayload), options);
-                await this.iteratePayloadAttributeOutput(`${entity.name}/`, messagePayload, options);
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
             } else if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(entity.name, JSON.stringify(messagePayload), options);
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
             } else {
                 /* istanbul ignore else */
                 if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${entity.name}/`, messagePayload, options);
+                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
                 }
             }
         }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {

lib/eventBus.js

@@ -6,6 +6,8 @@ const allowedEvents = [
     'deviceRenamed', // Device has been renamed
     'groupRenamed', // Group has been renamed
     'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
+    'groupMembersChanged', // Members of a group has been changed
 ];
 
 class EventBus extends events.EventEmitter {

lib/extension/availability.js

@@ -2,26 +2,21 @@ const logger = require('../util/logger');
 const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-// Some EndDevices should be pinged
+// Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableModels = [
-    'E11-G13',
-    '53170161',
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
-const forcedPingable =
-    zigbeeHerdsmanConverters.devices.filter((d) => pingableModels.includes(d.model));
-
-const toZigbeeCandidates = ['state', 'brightness', 'color', 'color_temp'];
-
 const Hours25 = 1000 * 60 * 60 * 25;
 
 /**
  * This extensions pings devices to check if they are online.
  */
-class DeviceAvailability extends BaseExtension {
+class Availability extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -29,30 +24,27 @@ class DeviceAvailability extends BaseExtension {
         this.timers = {};
         this.state = {};
 
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
 
-        // Initialize whitelist
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => {
-            return settings.getEntity(e).ID;
-        });
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
     }
 
-    isAllowed(device) {
+    inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
-
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : null;
+        const name = deviceSettings.friendlyName;
 
-        // Whitelist is not empty and device is in it, enable availability
-        if (this.whitelist.length > 0) {
-            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
         }
 
-        // Device is on blacklist, disable availability
-        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
             return false;
         }
 
@@ -60,27 +52,25 @@ class DeviceAvailability extends BaseExtension {
     }
 
     isPingable(device) {
-        // Device is on forcedPingable-list, enable availability
-        if (forcedPingable.find((d) => d.zigbeeModel.includes(device.modelID))) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
             return true;
         }
 
         // Device is a mains powered router
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-
-        return result;
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
     }
 
     onMQTTConnected() {
         for (const device of this.zigbee.getClients()) {
             // Mark all devices as online on start
-            this.publishAvailability(device, true);
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.isAllowed(device)) {
+            if (this.inPasslistOrNotInBlocklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
-                    this.timers[device.ieeeAddr] = setInterval(() => {
+                    this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
                     }, utils.secondsToMilliseconds(300));
                 }
@@ -90,34 +80,33 @@ class DeviceAvailability extends BaseExtension {
 
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
             logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
         }
 
-        const ieeeAddr = device.ieeeAddr;
-        const level = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
         try {
             await device.ping();
             this.publishAvailability(device, true);
-            logger.debug(`Successfully pinged '${entity.name}'`);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
         } catch (error) {
             this.publishAvailability(device, false);
-            logger[level](`Failed to ping '${entity.name}'`);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
         } finally {
             this.setTimerPingable(device);
         }
     }
 
     async handleIntervalNotPingable(device) {
-        const ago = Date.now() - device.lastSeen;
-        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!entity || !device.lastSeen) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
             return;
         }
 
-        logger.debug(`Non-pingable device '${entity.name}' was last seen '${ago / 1000}' seconds ago.`);
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
 
         if (ago > Hours25) {
             this.publishAvailability(device, false);
@@ -143,50 +132,46 @@ class DeviceAvailability extends BaseExtension {
     }
 
     async onReconnect(device) {
-        if (device && device.modelID) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-
-            if (mappedDevice) {
-                const used = [];
-                try {
-                    for (const key of toZigbeeCandidates) {
-                        const converter = mappedDevice.toZigbee.find((tz) => tz.key.includes(key));
-                        if (converter && !used.includes(converter)) {
-                            await converter.convertGet(device.endpoints[0], key, {});
-                            used.push(converter);
-                        }
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            const used = [];
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter && !used.includes(converter)) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                        used.push(converter);
                     }
-                } catch (error) {
-                    const entity = this.zigbee.resolveEntity(device.ieeeAddr);
-                    logger.error(`Failed to read state of '${entity.name}' after reconnect`);
                 }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
             }
         }
     }
 
-    publishAvailability(device, available) {
+    publishAvailability(device, available, force=false) {
         const ieeeAddr = device.ieeeAddr;
         if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
             this.onReconnect(device);
         }
 
         const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
-        if (this.state[ieeeAddr] !== available) {
+        if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
             this.mqtt.publish(topic, payload, {retain: true, qos: 0});
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
         const device = data.device;
         if (!device) {
             return;
         }
 
-        if (this.isAllowed(device)) {
+        if (this.inPasslistOrNotInBlocklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {
@@ -212,4 +197,4 @@ class DeviceAvailability extends BaseExtension {
     }
 }
 
-module.exports = DeviceAvailability;
+module.exports = Availability;

lib/extension/bind.js

@@ -0,0 +1,159 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
+const Extension = require('./extension');
+
+const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+
+// See zigbee-herdsman-converters devices.js
+const defaultBindGroup = {type: 'group_number', ID: 901};
+
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
+        }
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        } else if (settings.get().experimental.new_api && topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
+        }
+
+        return {type, sourceKey, targetKey};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
+
+        let error = null;
+        const source = this.zigbee.resolveEntity(sourceKey);
+        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        const responseData = {from: sourceKey, to: targetKey};
+
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
+
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusters) {
+                const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+
+                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
+
+                    try {
+                        let bindTarget = null;
+                        if (target.type === 'group') bindTarget = target.group;
+                        else if (target.type === 'group_number') bindTarget = target.ID;
+                        else bindTarget = target.endpoint;
+
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    }
+                }
+            }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
+        }
+
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, JSON.stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        }
+    }
+}
+
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,390 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.lastJoinedDeviceIeeeAddr = null;
+
+        this.requestLookup = {
+            'device/options': this.deviceOptions.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/rename': this.deviceRename.bind(this),
+            'group/add': this.groupAdd.bind(this),
+            'group/options': this.groupOptions.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+            'group/rename': this.groupRename.bind(this),
+            'permit_join': this.permitJoin.bind(this),
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
+            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'health_check': this.healthCheck.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
+        }
+
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') {
+                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
+            } else if (type === 'deviceInterview') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
+                };
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish(
+                'bridge/event',
+                JSON.stringify({type: utils.toSnakeCase(type), data: payload}),
+                {retain: false, qos: 0},
+            );
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    async deviceOptions(message) {
+        return this.changeEntityOptions('device', message);
+    }
+
+    async groupOptions(message) {
+        return this.changeEntityOptions('group', message);
+    }
+
+    async deviceRemove(message) {
+        return this.removeEntity('device', message);
+    }
+
+    async groupRemove(message) {
+        return this.removeEntity('group', message);
+    }
+
+    async healthCheck(message) {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    async groupAdd(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    async deviceRename(message) {
+        return this.renameEntity('device', message);
+    }
+
+    async groupRename(message) {
+        return this.renameEntity('group', message);
+    }
+
+    async permitJoin(message) {
+        const value = this.getValue(message);
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    configLastSeen(message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configElapsed(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configLogLevel(message) {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    async touchlinkFactoryReset(message) {
+        logger.info('Start touchlink factory reset');
+        const result = await this.zigbee.touchlinkFactoryReset();
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, {}, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    /**
+     * Utils
+     */
+
+    getValue(message) {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
+
+            return message.value;
+        } else {
+            return message;
+        }
+    }
+
+    changeEntityOptions(entityType, message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        settings.changeEntityOptions(ID, message.options);
+        const cleanup = (o) => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+        const oldOptions = cleanup(entity.settings);
+        const newOptions = cleanup(settings.getEntity(ID));
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
+
+    renameEntity(entityType, message) {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const entity = this.getEntity(entityType, from);
+
+        settings.changeFriendlyName(from, to);
+
+        if (entity.type === 'device') {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device});
+        } else {
+            this.publishGroups();
+            this.eventBus.emit(`groupRenamed`, {group: entity.group});
+        }
+
+        return utils.getResponse(message, {from: entity.settings.friendlyName, to}, null);
+    }
+
+    async removeEntity(entityType, message) {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity.type === 'device') {
+                if (block) {
+                    settings.blockDevice(entity.settings.ID);
+                }
+
+                if (force) {
+                    await entity.device.removeFromDatabase();
+                } else {
+                    await entity.device.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    await entity.group.removeFromDatabase();
+                } else {
+                    await entity.group.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {device: entity.device});
+            }
+
+            // Remove from configuration.yaml
+            if (entity.type === 'device') {
+                settings.removeDevice(entity.settings.ID);
+            } else {
+                settings.removeGroup(entity.settings.ID);
+            }
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+
+            if (entity.type === 'device') {
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type, ID) {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
+
+    async publishInfo() {
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices() {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieee_address: device.ieeeAddr,
+                type: device.type,
+                network_address: device.networkAddress,
+                supported: !!definition,
+                friendly_name: resolved.name,
+                definition: definitionPayload,
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
+                interviewing: device.interviewing,
+                interview_completed: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups() {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                id: group.groupID,
+                friendly_name: resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieee_address: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = Bridge;

lib/extension/configure.js

@@ -0,0 +1,144 @@
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
+        this.attempts = {};
+
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+    }
+
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
+
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.mqtt.subscribe(this.legacyTopic);
+        }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (topic === this.legacyTopic) {
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
+
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(resolvedEntity, true);
+        } else if (topic === this.topic) {
+            message = utils.parseJSON(message, message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const resolvedEntity = this.zigbee.resolveEntity(ID);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                error = `Device '${ID}' does not exist`;
+            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                error = `Device '${resolvedEntity.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(resolvedEntity, true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, JSON.stringify(response));
+        }
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    async configure(resolvedEntity, force=false, thowError=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+        } catch (error) {
+            this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/deviceBind.js

@@ -1,81 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-const BaseExtension = require('./baseExtension');
-
-const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
-
-// See zigbee-herdsman-converters devices.js
-const defaultBindGroup = {type: 'group_number', ID: 901};
-
-class DeviceBind extends BaseExtension {
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Parse topic, retrieve type (bind or unbind) and source
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-        const type = topic.split('/')[0];
-        const sourceKey = topic.replace(`${type}/`, '');
-        const targetKey = message;
-
-        // Find source; can only be a device and target
-        const source = this.zigbee.resolveEntity(sourceKey);
-        assert(source != null && source.type === 'device', 'Source undefined or not a device');
-        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
-        assert(target != null, 'Target is unknown');
-
-        const sourceName = source.settings.friendlyName;
-        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
-
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters.
-        for (const cluster of clusters) {
-            const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
-
-            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                try {
-                    let bindTarget = null;
-                    if (target.type === 'group') bindTarget = target.group;
-                    else if (target.type === 'group_number') bindTarget = target.ID;
-                    else bindTarget = target.endpoint;
-
-                    if (type === 'bind') {
-                        await source.endpoint.bind(cluster, bindTarget);
-                    } else {
-                        await source.endpoint.unbind(cluster, bindTarget);
-                    }
-
-                    logger.info(
-                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceName}' to '${targetName}'`,
-                    );
-                    this.mqtt.log(
-                        `device_${type}`,
-                        {from: sourceName, to: targetName, cluster},
-                    );
-                } catch (error) {
-                    logger.error(
-                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                        `'${targetName}' (${error})`,
-                    );
-                    this.mqtt.log(
-                        `device_${type}_failed`,
-                        {from: sourceName, to: targetName, cluster},
-                    );
-                }
-            }
-        }
-    }
-}
-
-module.exports = DeviceBind;

lib/extension/deviceConfigure.js

@@ -1,112 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const BaseExtension = require('./baseExtension');
-
-class DeviceConfigure extends BaseExtension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        this.configuring = new Set();
-        this.attempts = {};
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-    }
-
-    shouldConfigure(device, mappedDevice) {
-        if (!device || !mappedDevice || !mappedDevice.configure) {
-            return false;
-        }
-
-        if (device.meta &&
-            device.meta.hasOwnProperty('configured') &&
-            device.meta.configured === mappedDevice.meta.configureKey) {
-            return false;
-        }
-
-        if (device.interviewing === true) {
-            return false;
-        }
-
-        return true;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (topic !== this.topic) {
-            return;
-        }
-
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || entity.type !== 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            return;
-        }
-
-        const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(entity.device.modelID);
-        if (!mappedDevice.configure) {
-            logger.warn(`Skipping configure of '${entity.name}', device does not require this.`);
-            return;
-        }
-
-        this.configure(entity.device, mappedDevice, entity.settings, true);
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            const settingsDevice = settings.getDevice(device.ieeeAddr);
-            if (this.shouldConfigure(device, mappedDevice)) {
-                await this.configure(device, mappedDevice, settingsDevice);
-            }
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        const device = data.device;
-
-        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(device, mappedDevice)) {
-            this.configure(device, mappedDevice, settingsDevice);
-        }
-    }
-
-    async configure(device, mappedDevice, settingsDevice, force=false) {
-        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
-            return false;
-        }
-
-        this.configuring.add(device.ieeeAddr);
-
-        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
-            this.attempts[device.ieeeAddr] = 0;
-        }
-
-        logger.info(`Configuring '${settingsDevice.friendlyName}'`);
-        try {
-            await mappedDevice.configure(device, this.coordinatorEndpoint);
-            logger.info(`Successfully configured '${settingsDevice.friendlyName}'`);
-            // eslint-disable-next-line
-            device.meta.configured = mappedDevice.meta.configureKey;
-            device.save();
-        } catch (error) {
-            logger.error(
-                `Failed to configure '${settingsDevice.friendlyName}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
-            this.attempts[device.ieeeAddr]++;
-        }
-
-        this.configuring.delete(device.ieeeAddr);
-    }
-}
-
-module.exports = DeviceConfigure;

lib/extension/deviceEvent.js

@@ -1,36 +0,0 @@
-const BaseExtension = require('./baseExtension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-class DeviceEvent extends BaseExtension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (data.device) {
-            this.callOnEvent(data.device, type, data, mappedDevice);
-        }
-    }
-
-    async stop() {
-        for (const device of this.zigbee.getClients()) {
-            this.callOnEvent(device, 'stop', {});
-        }
-    }
-
-    callOnEvent(device, type, data, mappedDevice) {
-        zigbeeHerdsmanConverters.onEvent(type, data, device);
-
-        if (!mappedDevice) {
-            mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-        }
-
-        if (mappedDevice && mappedDevice.onEvent) {
-            mappedDevice.onEvent(type, data, device);
-        }
-    }
-}
-
-module.exports = DeviceEvent;

lib/extension/extension.js

@@ -1,4 +1,4 @@
-class BaseExtension {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -30,10 +30,10 @@ class BaseExtension {
      * Is called when a Zigbee message from a device is received.
      * @param {string} type Type of the message
      * @param {Object} data Data of the message
-     * @param {Object?} mappedDevice The mapped device
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
      * @param {Object?} settingsDevice Device settings
      */
-    // onZigbeeEvent(type, data, mappedDevice, settingsDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -49,4 +49,4 @@ class BaseExtension {
     // stop() {}
 }
 
-module.exports = BaseExtension;
+module.exports = Extension;

lib/extension/externalConverters.js

@@ -0,0 +1,18 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+
+class ExternalConverters extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/groups.js

@@ -1,30 +1,45 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups extends BaseExtension {
+class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
 
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}remove`);
-            this.mqtt.subscribe(`${topic}add`);
-            this.mqtt.subscribe(`${topic}remove_all`); // DEPRECATED
+            for (let step = 1; step < 20; step++) {
+                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+                this.mqtt.subscribe(`${topic}remove`);
+                this.mqtt.subscribe(`${topic}add`);
+                this.mqtt.subscribe(`${topic}remove_all`);
+            }
+        }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
         }
     }
 
     async onZigbeeStarted() {
         await this.syncGroupsWithSettings();
-        this.state.on('stateChange', this.onStateChange);
+        this.eventBus.on('stateChange', this.onStateChange);
     }
 
     async syncGroupsWithSettings() {
@@ -85,33 +100,33 @@ class Groups extends BaseExtension {
         });
 
         if (Object.keys(payload).length) {
-            const entity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
                 return settingsGroup && settingsGroup.optimistic;
             });
 
-            if (entity.type === 'device') {
+            if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(entity.endpoint)) {
-                        if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
                 }
             } else {
                 const groupIDsToPublish = new Set();
-                for (const member of entity.group.members) {
+                for (const member of resolvedEntity.group.members) {
                     await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
                                 groupIDsToPublish.add(zigbeeGroup.groupID);
                             }
                         }
                     }
                 }
-                groupIDsToPublish.delete(entity.group.groupID);
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
                 for (const groupID of groupIDsToPublish) {
                     await this.publishEntityState(groupID, payload, reason);
                 }
@@ -119,7 +134,7 @@ class Groups extends BaseExtension {
         }
     }
 
-    allMembersOff(zigbeeGroup) {
+    areAllMembersOff(zigbeeGroup) {
         for (const member of zigbeeGroup.members) {
             const device = member.getDevice();
             if (this.state.exists(device.ieeeAddr)) {
@@ -132,68 +147,184 @@ class Groups extends BaseExtension {
         return true;
     }
 
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
+
+        /* istanbul ignore else */
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else {
+                type = 'remove_all';
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
+        }
+
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi,
+        };
+    }
+
     async onMQTTMessage(topic, message) {
-        let type;
-        let group;
-        const topicMatch = topic.match(topicRegex);
-        if (topicMatch) {
-            group = this.zigbee.resolveEntity(topicMatch[1]);
-            type = topicMatch[2];
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey,
+        } = this.parseMQTTMessage(topic, message);
+        if (!type) return;
 
-            if (!group || group.type !== 'group') {
-                logger.error(`Group '${topicMatch[1]}' does not exist`);
-                this.mqtt.log(
-                    `device_group_${type}_failed`,
-                    {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'});
-                return;
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
+        }
+
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
+
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                const endpointName = endpoints ?
+                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+                }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
             }
-        } else if (topic.match(topicRegexRemoveAll)) {
-            type = 'remove_all';
+        }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, JSON.stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
         } else {
-            return;
-        }
-
-        const entity = this.zigbee.resolveEntity(message);
-        if (!entity || !entity.type === 'device') {
-            logger.error(`Device '${message}' does not exist`);
-            this.mqtt.log(
-                `device_group_${type}_failed`,
-                {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'});
-            return;
-        }
-
-        const keys = [
-            `${entity.device.ieeeAddr}/${entity.endpoint.ID}`,
-            `${entity.name}/${entity.endpoint.ID}`,
-        ];
-
-        if (entity.endpointName) {
-            keys.push(`${entity.device.ieeeAddr}/${entity.endpointName}`);
-            keys.push(`${entity.name}/${entity.endpointName}`);
-        }
-
-        if (entity.isDefaultEndpoint) {
-            keys.push(entity.name);
-            keys.push(entity.device.ieeeAddr);
-        }
-
-        if (type === 'add') {
-            logger.info(`Adding '${entity.name}' to '${group.name}'`);
-            await entity.endpoint.addToGroup(group.group);
-            settings.addDeviceToGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_add', {friendly_name: entity.name, group: group.name});
-        } else if (type === 'remove') {
-            logger.info(`Removing '${entity.name}' from '${group.name}'`);
-            await entity.endpoint.removeFromGroup(group.group);
-            settings.removeDeviceFromGroup(group.settings.ID, keys);
-            this.mqtt.log('device_group_remove', {friendly_name: entity.name, group: group.name});
-        } else { // remove_all
-            logger.info(`Removing '${entity.name}' from all groups`);
-            await entity.endpoint.removeFromAllGroups();
-            for (const settingsGroup of settings.getGroups()) {
-                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-                this.mqtt.log('device_group_remove_all', {friendly_name: entity.name});
-            }
+            this.eventBus.emit('groupMembersChanged', {group: resolvedEntityGroup});
         }
     }
 }

lib/extension/legacy/bridgeLegacy.js

@@ -1,15 +1,15 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const utils = require('../util/utils');
+const utils = require('../../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('../extension');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
-class BridgeConfig extends BaseExtension {
+class BridgeLegacy extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
@@ -52,6 +52,7 @@ class BridgeConfig extends BaseExtension {
             'device_options': this.deviceOptions,
             'add_group': this.addGroup,
             'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
             'whitelist': this.whitelist,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
         };
@@ -63,7 +64,10 @@ class BridgeConfig extends BaseExtension {
             assert(entity, `Entity '${message}' does not exist`);
             settings.whitelistDevice(entity.ID);
             logger.info(`Whitelisted '${entity.friendlyName}'`);
-            this.mqtt.log('device_whitelisted', {friendly_name: entity.friendlyName});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
         }
@@ -85,7 +89,7 @@ class BridgeConfig extends BaseExtension {
 
         const entity = settings.getEntity(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
-        settings.changeDeviceOptions(entity.ID, json.options);
+        settings.changeEntityOptions(entity.ID, json.options);
         logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
     }
 
@@ -147,11 +151,11 @@ class BridgeConfig extends BaseExtension {
             };
 
             if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
                 const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelID;
-                payload.vendor = mappedDevice ? mappedDevice.vendor : '-';
-                payload.description = mappedDevice ? mappedDevice.description : '-';
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
                 payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
                 payload.manufacturerID = device.manufacturerID;
                 payload.manufacturerName = device.manufacturerName;
@@ -174,20 +178,23 @@ class BridgeConfig extends BaseExtension {
         if (topic.split('/').pop() == 'get') {
             this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
         } else {
-            this.mqtt.log('devices', devices);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
         }
     }
 
     groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups().map((g) => {
+        const payload = settings.getGroups().map((g) => {
             const group = {...g};
             delete group.friendlyName;
             return group;
-        }));
+        });
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} got ${message}`;
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
 
         let json = null;
         try {
@@ -223,7 +230,11 @@ class BridgeConfig extends BaseExtension {
             const entity = this.zigbee.resolveEntity(to);
             const eventData = isGroup ? {group: entity.group} : {device: entity.device};
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
-            this.mqtt.log(`${isGroup ? 'group' : 'device'}_renamed`, {from, to});
+
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
         }
@@ -254,7 +265,7 @@ class BridgeConfig extends BaseExtension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
-        this.mqtt.log('group_added', name);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
@@ -262,9 +273,15 @@ class BridgeConfig extends BaseExtension {
         const name = message;
         const entity = this.zigbee.resolveEntity(message);
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
         settings.removeGroup(message);
-        entity.group.removeFromDatabase();
-        this.mqtt.log('group_removed', message);
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -291,7 +308,7 @@ class BridgeConfig extends BaseExtension {
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
 
-            this.mqtt.log(`device_${lookup[action][0]}_failed`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
@@ -306,7 +323,7 @@ class BridgeConfig extends BaseExtension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.log(`device_${lookup[action][0]}`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -323,7 +340,7 @@ class BridgeConfig extends BaseExtension {
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
-            this.mqtt.log(`device_${lookup[action][0]}_failed`, message);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -368,25 +385,86 @@ class BridgeConfig extends BaseExtension {
         await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
         if (type === 'deviceJoined') {
-            this.lastJoinedDeviceName = settingsDevice.friendlyName;
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
         }
     }
 
     async touchlinkFactoryReset() {
         logger.info('Starting touchlink factory reset...');
-        this.mqtt.log('touchlink', 'reset_started', {status: 'started'});
+        this.mqtt.publish(
+            'bridge/log',
+            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
         const result = await this.zigbee.touchlinkFactoryReset();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
-            this.mqtt.log('touchlink', 'reset_success', {status: 'success'});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
-            this.mqtt.log('touchlink', 'reset_failed', {status: 'failed'});
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
         }
     }
 }
 
-module.exports = BridgeConfig;
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -1,11 +1,11 @@
 /* istanbul ignore file */
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
 
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
-class DeviceGroupMembership extends BaseExtension {
+class DeviceGroupMembership extends Extension {
     onMQTTConnected() {
         for (let step = 1; step < 20; step++) {
             const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;

lib/extension/legacy/softReset.js

@@ -1,14 +1,14 @@
 /* istanbul ignore file */
 // DEPRECATED
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset extends BaseExtension {
+class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;

lib/extension/networkMap.js

@@ -1,39 +1,78 @@
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class NetworkMap extends BaseExtension {
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
 
         // Bind
         this.raw = this.raw.bind(this);
         this.graphviz = this.graphviz.bind(this);
+        this.plantuml = this.plantuml.bind(this);
 
         // Set supported formats
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
         };
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-        this.mqtt.subscribe(this.topicRoutes);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+            this.mqtt.subscribe(this.legacyTopicRoutes);
+        }
+
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        if ((topic === this.topic || topic === this.topicRoutes) && this.supportedFormats.hasOwnProperty(message)) {
-            const includeRoutes = topic === this.topicRoutes;
-            const topology = await this.networkScan(includeRoutes);
-            const converted = this.supportedFormats[message](topology);
-            this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                const converted = this.supportedFormats[message](topology);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
+        }
+
+        if (settings.get().experimental.new_api && topic === this.topic) {
+            try {
+                message = utils.parseJSON(message, message);
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    JSON.stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    JSON.stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
         }
     }
 
@@ -47,43 +86,43 @@ class NetworkMap extends BaseExtension {
         let text = 'digraph G {\nnode[shape=record];\n';
         let style = '';
 
-        topology.nodes.forEach((device) => {
+        topology.nodes.forEach((node) => {
             const labels = [];
 
             // Add friendly name
-            labels.push(`${device.friendlyName}`);
+            labels.push(`${node.friendlyName}`);
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${device.ieeeAddr} (${device.networkAddress})` +
-                ((device.failed && device.failed.length) ? `failed: ${device.failed.join(',')}` : ''),
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
-            if (device.type !== 'Coordinator') {
-                const mappedModel = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                if (mappedModel) {
-                    labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+            if (node.type !== 'Coordinator') {
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
-                    labels.push(`${device.manufacturerName} ${device.modelID}`);
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
                 }
             }
 
             // Add the device last_seen timestamp
             let lastSeen = 'unknown';
-            const date = device.type === 'Coordinator' ? Date.now() : device.lastSeen;
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+                lastSeen = utils.formatDate(date, 'relative');
             }
 
             labels.push(lastSeen);
 
             // Shape the record according to device type
-            if (device.type == 'Coordinator') {
+            if (node.type == 'Coordinator') {
                 style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
                     `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
+            } else if (node.type == 'Router') {
                 style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
                     `fontcolor="${colors.font.router}"`;
             } else {
@@ -92,22 +131,22 @@ class NetworkMap extends BaseExtension {
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
              * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.links.filter((e) => (e.source.ieeeAddr === device.ieeeAddr)).forEach((e) => {
-                const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' :
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
                 const textRoutes = e.routes.map((r) => r.destinationAddress);
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
-                text += `  "${device.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
                 text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
             });
         });
@@ -117,33 +156,91 @@ class NetworkMap extends BaseExtension {
         return text.replace(/\0/g, '');
     }
 
+    plantuml(topology) {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.getDevices();
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
         const lqis = new Map();
         const routingTables = new Map();
         const failed = new Map();
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             try {
                 const result = await device.lqi();
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolved.name}'`);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolved.name}'`);
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
             }
 
             if (includeRoutes) {
                 try {
                     const result = await device.routingTable();
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolved.name}'`);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolved.name}'`);
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
                 }
             }
         }
@@ -153,9 +250,9 @@ class NetworkMap extends BaseExtension {
         const networkMap = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolved = this.zigbee.resolveEntity(device);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
             networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolved.name, type: device.type,
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
             });

lib/extension/onEvent.js

@@ -0,0 +1,37 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -1,26 +1,39 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
-const BaseExtension = require('./baseExtension');
+const utils = require('../util/utils');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
+const Extension = require('./extension');
 const MINUTES_10 = 1000 * 60 * 10;
 
-class OTAUpdate extends BaseExtension {
+class OTAUpdate extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.inProgress = new Set();
         this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
+        }
     }
 
-    async onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (data.type !== 'commandQueryNextImageRequest') return;
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
 
-        const supportsOTA = mappedDevice && mappedDevice.hasOwnProperty('ota');
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
             // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
@@ -30,13 +43,21 @@ class OTAUpdate extends BaseExtension {
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
-            const available = await mappedDevice.ota.isUpdateAvailable(data.device, logger, data.data);
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
             this.publishEntityState(data.device.ieeeAddr, {update_available: available});
 
             if (available) {
-                const message = `Update available for '${settingsDevice.friendly_name}'`;
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
                 logger.info(message);
-                this.mqtt.log('ota_update', message, {status: 'available', device: settingsDevice.friendly_name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
             }
         }
 
@@ -44,7 +65,10 @@ class OTAUpdate extends BaseExtension {
         // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
         // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
         const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
-        await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
     }
 
     async readSoftwareBuildIDAndDateCode(device, update) {
@@ -65,76 +89,147 @@ class OTAUpdate extends BaseExtension {
     }
 
     async onMQTTMessage(topic, message) {
-        if (!topic.match(topicRegex)) {
+        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
             return null;
         }
 
-        const device = this.zigbee.resolveEntity(message);
-        assert(device != null && device.type === 'device', 'Device not found or not a device');
-        if (!device.mapped || !device.mapped.ota) {
-            const message = `Device '${device.name}' does not support OTA updates`;
-            logger.error(message);
-            this.mqtt.log('ota_update', message, {status: `not_supported`, device: device.name});
-            return;
-        }
+        message = utils.parseJSON(message, message);
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+        const resolvedEntity = this.zigbee.resolveEntity(ID);
+        const type = topic.substring(topic.lastIndexOf('/') + 1);
+        const responseData = {id: ID};
+        let error = null;
 
-        if (this.inProgress.has(device.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${device.name}', skipping...`);
-            return;
-        }
-        this.inProgress.add(device.device.ieeeAddr);
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            error = `Device '${ID}' does not exist`;
+        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
 
-        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
-        if (type === 'check') {
-            const message = `Checking if update available for '${device.name}'`;
-            logger.info(message);
-            this.mqtt.log('ota_update', message, {status: `checking_if_available`, device: device.name});
-            try {
-                const available = await device.mapped.ota.isUpdateAvailable(device.device, logger);
-                const message=(available ?
-                    `Update available for '${device.name}'` : `No update available for '${device.name}'`);
-                logger.info(message);
-                const meta = {status: available ? 'available' : 'not_available', device: device.name};
-                this.mqtt.log('ota_update', message, meta);
-                this.publishEntityState(device.device.ieeeAddr, {update_available: available});
-                this.lastChecked[device.device.ieeeAddr] = Date.now();
-            } catch (error) {
-                const message = `Failed to check if update available for '${device.name}' (${error.message})`;
-                logger.error(message);
-                this.mqtt.log('ota_update', message, {status: `check_failed`, device: device.name});
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message: error, meta}),
+                );
             }
-        } else { // type === 'update'
-            const message = `Updating '${device.name}' to latest firmware`;
-            logger.info(message);
-            this.mqtt.log('ota_update', message, {status: `update_in_progress`, device: device.name});
-            try {
-                const onProgress = (progress, remaining) => {
-                    let message = `Update of '${device.name}' at ${progress}%`;
-                    if (remaining) {
-                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else {
+            this.inProgress.add(resolvedEntity.device.ieeeAddr);
+
+            if (type === 'check') {
+                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
+                        resolvedEntity.device, logger,
+                    );
+                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    logger.info(msg);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `ota_update`, message: msg, meta}),
+                        );
                     }
 
-                    logger.info(message);
-                    this.mqtt.log('ota_update', message, {status: `update_progress`, device: device.name, progress});
-                };
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(device.device, false);
-                await device.mapped.ota.updateToLatest(device.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(device.device, true);
-                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${device.name}'` + (to ? `, from '${fromS}' to '${toS}'` : ``);
-                logger.info(message);
-                const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
-                this.mqtt.log('ota_update', message, meta);
-                this.publishEntityState(device.device.ieeeAddr, {update_available: false});
-            } catch (error) {
-                const message = `Update of '${device.name}' failed (${error.message})`;
-                logger.error(message);
-                this.mqtt.log('ota_update', message, {status: `update_failed`, device: device.name});
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
+                }
+            } else { // type === 'update'
+                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress, remaining) => {
+                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    }
+                } catch (e) {
+                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message: error, meta}));
+                    }
+                }
             }
+
+            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
         }
 
-        this.inProgress.delete(device.device.ieeeAddr);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, JSON.stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        }
     }
 }
 

lib/extension/publish.js

@@ -4,10 +4,9 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
@@ -19,9 +18,10 @@ const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
 ];
 
-class EntityPublish extends BaseExtension {
+class EntityPublish extends Extension {
     onMQTTConnected() {
         // Subscribe to topics.
         const baseTopic = settings.get().mqtt.base_topic;
@@ -46,7 +46,7 @@ class EntityPublish extends BaseExtension {
             return null;
         }
 
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
     }
 
     async onMQTTMessage(topic, message) {
@@ -55,12 +55,20 @@ class EntityPublish extends BaseExtension {
             return false;
         }
 
-        const entityName = `${topic.ID}` + (topic.postfix ? `/${topic.postfix}` : '');
-        const entity = this.zigbee.resolveEntity(entityName);
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
 
-        if (!entity) {
-            this.mqtt.log('entity_not_found', {friendly_name: entityName});
-            logger.error(`Entity '${entityName}' is unknown`);
+        if (!resolvedEntity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityKey};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityKey}' is unknown`);
             return;
         }
 
@@ -69,26 +77,26 @@ class EntityPublish extends BaseExtension {
         let target = null;
         let options = {};
         let device = null;
-        let mapped = null;
+        let definition = null;
 
-        assert(entity.type === 'device' || entity.type === 'group');
-        if (entity.type === 'device') {
-            // Map device to a model
-            if (!entity.mapped) {
-                logger.warn(`Device with modelID '${entity.device.modelID}' is not supported.`);
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
                 logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
                 return;
             }
 
-            device = entity.device;
-            mapped = entity.mapped;
-            target = entity.endpoint;
-            converters = entity.mapped.toZigbee;
-            options = entity.settings;
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = {...settings.get().device_options, ...resolvedEntity.settings};
         } else {
             converters = groupConverters;
-            target = entity.group;
-            options = entity.settings;
+            target = resolvedEntity.group;
+            options = {...settings.get().device_options, ...resolvedEntity.settings};
+            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
         }
 
         // Convert the MQTT message to a Zigbee message.
@@ -104,12 +112,13 @@ class EntityPublish extends BaseExtension {
             }
         }
 
+
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(entity.settings.ID) || {};
+        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
@@ -121,31 +130,47 @@ class EntityPublish extends BaseExtension {
             }
         }
 
-        // Ensure that state and brightness are executed before other commands.
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
         const entries = Object.entries(json);
-        entries.sort((a, b) => (['state', 'brightness'].includes(a[0]) ? -1 : 1));
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
 
         // For each attribute call the corresponding converter
-        const usedConverters = [];
+        const usedConverters = {};
         for (let [key, value] of entries) {
-            let postfix = topic.postfix;
+            let endpointName = topic.endpointName;
             let actualTarget = target;
 
-            // When the key has a postfix included (e.g. state_right), this will override the target.
-            if (entity.type === 'device' && key.includes('_')) {
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
                 const underscoreIndex = key.lastIndexOf('_');
-                const possiblePostfix = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getPostfixes().includes(possiblePostfix)) {
-                    postfix = possiblePostfix;
+                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possibleEndpointName)) {
+                    endpointName = possibleEndpointName;
                     key = key.substring(0, underscoreIndex);
                     const device = target.getDevice();
-                    actualTarget = device.getEndpoint(mapped.endpoint(device)[postfix]);
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
                 }
             }
 
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters.includes(converter)) {
+            if (usedConverters[endpointOrGroupID].includes(converter)) {
                 // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
@@ -157,32 +182,32 @@ class EntityPublish extends BaseExtension {
 
             // Converter didn't return a result, skip
             const meta = {
-                endpoint_name: postfix,
+                endpoint_name: endpointName,
                 options,
                 message: json,
                 logger,
                 device,
                 state: deviceState,
-                mapped,
+                mapped: definition,
             };
 
             try {
                 if (topic.type === 'set' && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
                     if (result && result.state) {
                         const msg = result.state;
 
-                        if (postfix) {
-                            for (const key of ['state', 'brightness']) {
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                                 if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${postfix}`] = msg[key];
+                                    msg[`${key}_${endpointName}`] = msg[key];
                                     delete msg[key];
                                 }
                             }
                         }
 
-                        this.publishEntityState(entity.settings.ID, msg);
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
                     }
 
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
@@ -193,13 +218,13 @@ class EntityPublish extends BaseExtension {
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
                     if (
-                        entity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        entity.settings.retrieve_state
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
                     ) {
                         setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
                     }
                 } else if (topic.type === 'get' && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${entity.name}'`);
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     await converter.convertGet(actualTarget, key, meta);
                 } else {
                     logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
@@ -207,13 +232,21 @@ class EntityPublish extends BaseExtension {
                 }
             } catch (error) {
                 const message =
-                    `Publish '${topic.type}' '${key}' to '${entity.name}' failed: '${error}'`;
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
                 logger.error(message);
                 logger.debug(error.stack);
-                this.mqtt.log('zigbee_publish_error', message, {friendly_name: entity.name});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
             }
 
-            usedConverters.push(converter);
+            usedConverters[endpointOrGroupID].push(converter);
         }
 
         return true;

lib/extension/receive.js

@@ -2,26 +2,40 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 
-class DeviceReceive extends BaseExtension {
+class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.coordinator = null;
         this.elapsed = {};
         this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
     }
 
     async onZigbeeStarted() {
         this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
     }
 
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
     publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
                 publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
                     this.debouncers[ieeeAddr].payload = {};
                 }, time * 1000),
             };
@@ -54,8 +68,8 @@ class DeviceReceive extends BaseExtension {
         return result;
     }
 
-    canHandleEvent(type, data, mappedDevice) {
-        if (type !== 'message') {
+    shouldProcess(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
             return false;
         }
 
@@ -77,47 +91,47 @@ class DeviceReceive extends BaseExtension {
          * As the same message is also received directly from the end device, it makes no sense
          * to handle these messages.
          */
-        const hasGroupID = data.hasOwnProperty('groupID') && data.groupID != 0;
+        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
         if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
             logger.debug('Skipping re-transmitted Xiaomi message');
             return false;
         }
 
-        if (data.device.modelID === null && data.device.interviewing) {
-            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
-            return false;
-        }
+        if (!resolvedEntity.definition) {
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
 
-        if (!mappedDevice) {
-            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
             return false;
         }
 
         return true;
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (!this.canHandleEvent(type, data, mappedDevice)) {
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
 
-        const converters = mappedDevice.fromZigbee.filter((c) => {
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
             const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
             return c.cluster === data.cluster && type;
         });
 
-        // Check if there is an available converter
-        if (!converters.length) {
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
-                `No converter available for '${mappedDevice.model}' with cluster '${data.cluster}' ` +
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
                 `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
             );
             return;
         }
 
-        const debounce = settingsDevice.debounce || settings.get().device_options.debounce;
-        const debounceIgnore = settingsDevice.debounce_ignore || settings.get().device_options.debounce_ignore;
+        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
+        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
 
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
@@ -144,29 +158,14 @@ class DeviceReceive extends BaseExtension {
                 this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant && settings.get().advanced.homeassistant_legacy_triggers) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(data.device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
             }
         };
 
-        const meta = {device: data.device};
+        const meta = {device: data.device, logger};
         let payload = {};
         converters.forEach((converter) => {
             const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, data, publish, options, meta);
+            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
             if (converted) {
                 payload = {...payload, ...converted};
             }
@@ -178,4 +177,4 @@ class DeviceReceive extends BaseExtension {
     }
 }
 
-module.exports = DeviceReceive;
+module.exports = Receive;

lib/extension/report.js

@@ -1,10 +1,8 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const CC2530Router = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
-const BASICZBR3 = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
 const utils = require('../util/utils');
-const BaseExtension = require('./baseExtension');
+const Extension = require('./extension');
 const debounce = require('debounce');
 const ZigbeeHerdsman = require('zigbee-herdsman');
 
@@ -12,10 +10,27 @@ const defaultConfiguration = {
     minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
 };
 
-const devicesNotSupportingReporting = [CC2530Router, BASICZBR3];
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+];
 
 const reportKey = 1;
 
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
 const clusters = {
     'genOnOff': [
         {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
@@ -24,9 +39,18 @@ const clusters = {
         {attribute: 'currentLevel', ...defaultConfiguration},
     ],
     'lightingColorCtrl': [
-        {attribute: 'colorTemperature', ...defaultConfiguration},
-        {attribute: 'currentX', ...defaultConfiguration},
-        {attribute: 'currentY', ...defaultConfiguration},
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
     ],
     'closuresWindowCovering': [
         {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
@@ -76,16 +100,16 @@ const pollOnMessage = [
     },
 ];
 
-class DeviceReport extends BaseExtension {
+class Report extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.configuring = new Set();
+        this.queue = new Set();
         this.failed = new Set();
         this.pollDebouncers = {};
     }
 
-    shouldIgnoreClusterForDevice(cluster, mappedDevice) {
-        if (mappedDevice === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
             // Device announces it but doesn't support it
             // https://github.com/Koenkk/zigbee2mqtt/issues/2611
             return true;
@@ -94,17 +118,28 @@ class DeviceReport extends BaseExtension {
         return false;
     }
 
-    async setupReporting(device, mappedDevice) {
-        if (this.configuring.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
-        this.configuring.add(device.ieeeAddr);
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
 
         try {
             for (const ep of device.endpoints) {
                 for (const [cluster, configuration] of Object.entries(clusters)) {
-                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, mappedDevice)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
                         logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                items.push({...entry});
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
                         await ep.bind(cluster, this.coordinatorEndpoint);
-                        await ep.configureReporting(cluster, configuration);
+                        await ep.configureReporting(cluster, items);
                         logger.info(
                             `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
                         );
@@ -123,12 +158,14 @@ class DeviceReport extends BaseExtension {
         }
 
         device.save();
-        this.configuring.delete(device.ieeeAddr);
+        this.queue.delete(device.ieeeAddr);
     }
 
-    shouldSetupReporting(mappedDevice, device, messageType) {
-        if (!device || !mappedDevice) return false;
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
 
+        const {device, definition} = resolvedEntity;
         // Handle messages of type endDeviceAnnce and devIncoming.
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
@@ -136,10 +173,11 @@ class DeviceReport extends BaseExtension {
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
         if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
 
+        if (resolvedEntity.device.interviewing === true) return false;
         if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
         if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
         // Gledopto devices don't support reporting.
-        if (devicesNotSupportingReporting.includes(mappedDevice) || mappedDevice.vendor === 'Gledopto') return false;
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
         return true;
     }
 
@@ -147,16 +185,16 @@ class DeviceReport extends BaseExtension {
         this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
 
         for (const device of this.zigbee.getClients()) {
-            const mappedDevice = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-            if (this.shouldSetupReporting(mappedDevice, device, null)) {
-                this.setupReporting(device, mappedDevice);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                await this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, mappedDevice, settingsDevice) {
-        if (this.shouldSetupReporting(mappedDevice, data.device, type)) {
-            this.setupReporting(data.device, mappedDevice);
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            await this.setupReporting(resolvedEntity);
         }
 
         if (type === 'message') {
@@ -182,7 +220,8 @@ class DeviceReport extends BaseExtension {
             let toPoll = [];
 
             // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) => e.binds.map((e) => e))));
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
             toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
             toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
             toPoll = toPoll.map((e) => e.target);
@@ -215,4 +254,4 @@ class DeviceReport extends BaseExtension {
     }
 }
 
-module.exports = DeviceReport;
+module.exports = Report;

lib/mqtt.js

@@ -95,11 +95,15 @@ class MQTT extends events.EventEmitter {
         this.emit('message', {topic, message: message.toString()});
     }
 
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
+
     async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
-        if (!this.client || this.client.reconnecting) {
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;
@@ -111,16 +115,6 @@ class MQTT extends events.EventEmitter {
             this.client.publish(topic, payload, options, () => resolve());
         });
     }
-
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
-    }
 }
 
 module.exports = MQTT;

lib/state.js

@@ -1,24 +1,23 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
+const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
-const events = require('events');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
     'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
-class State extends events.EventEmitter {
-    constructor() {
-        super();
+class State {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -55,9 +54,13 @@ class State extends events.EventEmitter {
     }
 
     save() {
-        logger.debug(`Saving state to file ${this.file}`);
-        const json = JSON.stringify(this.state, null, 4);
-        fs.writeFileSync(this.file, json, 'utf8');
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            fs.writeFileSync(this.file, json, 'utf8');
+        } else {
+            logger.debug(`Not saving state`);
+        }
     }
 
     exists(ID) {
@@ -70,17 +73,18 @@ class State extends events.EventEmitter {
 
     set(ID, state, reason=null) {
         const toState = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (toState.hasOwnProperty(property)) {
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
                 delete toState[property];
             }
-        });
+        }
 
         const fromState = this.state[ID];
 
         this.state[ID] = toState;
 
-        this.emit('stateChange', {ID, from: fromState, to: toState, reason});
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
     remove(ID) {

lib/util/logger.js

@@ -44,7 +44,8 @@ const transportsToUse = [
         format: winston.format.combine(
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
+                let {timestamp, level, message} = info;
+                level = level === 'warning' ? 'warn' : level;
                 const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
@@ -54,26 +55,40 @@ const transportsToUse = [
 
 // Add file logger when enabled
 // NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            let {timestamp, level, message} = info;
+            level = level === 'warning' ? 'warn' : level;
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
 if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File({
-        filename: path.join(directory, logFilename),
-        json: false,
-        level,
-        tailable: true,
-        maxFiles: 3, // Keep last 3 files
-        maxsize: 10000000, // 10MB
-        format: winston.format.combine(
-            winston.format.timestamp({format: timestampFormat}),
-            winston.format.printf(/* istanbul ignore next */(info) => {
-                const {timestamp, level, message} = info;
-                return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
-            }),
-        ),
-    }));
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
+    transportsToUse.push(new winston.transports.Syslog(options));
 }
 
 // Create logger
-const logger = winston.createLogger({transports: transportsToUse});
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -112,4 +127,8 @@ if (output.includes('file')) {
     logger.info(`Logging to console only'`);
 }
 
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
+
 module.exports = logger;

lib/util/settings.js

@@ -8,6 +8,9 @@ const Ajv = require('ajv');
 const ajv = new Ajv({allErrors: true});
 
 const defaults = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
     whitelist: [],
     ban: [],
     permit_join: false,
@@ -41,21 +44,27 @@ const defaults = {
     experimental: {
         // json or attribute or attribute_and_json
         output: 'json',
+        new_api: false,
     },
     advanced: {
+        legacy_api: true,
+        log_rotation: true,
         log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
         log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
-        baudrate: 115200,
-        rtscts: true,
+        adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
         availability_blacklist: [],
         availability_whitelist: [],
 
@@ -68,6 +77,8 @@ const defaults = {
          * https://www.zigbee2mqtt.io/configuration/configuration.html
          */
         cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
 
         /**
          * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
@@ -117,6 +128,7 @@ const defaults = {
          */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
+    external_converters: [],
 };
 
 const schema = {
@@ -148,23 +160,32 @@ const schema = {
             properties: {
                 port: {type: ['string', 'null']},
                 disable_led: {type: 'boolean'},
+                adapter: {type: 'string', enum: ['deconz', 'zstack']},
             },
         },
-        ban: {type: 'array', items: {type: 'string'}},
+        blocklist: {type: 'array', items: {type: 'string'}},
+        passlist: {type: 'array', items: {type: 'string'}},
+        // Deprecated: use block/passlist
         whitelist: {type: 'array', items: {type: 'string'}},
+        ban: {type: 'array', items: {type: 'string'}},
         experimental: {
             type: 'object',
             properties: {
                 transmit_power: {type: 'number'},
+                output: {type: 'string', enum: ['attribute_and_json', 'attribute', 'json']},
             },
         },
         advanced: {
             type: 'object',
             properties: {
+                legacy_api: {type: 'boolean'},
                 pan_id: {type: 'number'},
                 ext_pan_id: {type: 'array', items: {type: 'number'}},
                 channel: {type: 'number', minimum: 11, maximum: 26},
                 cache_state: {type: 'boolean'},
+                cache_state_persistent: {type: 'boolean'},
+                cache_state_send_on_startup: {type: 'boolean'},
+                log_rotation: {type: 'boolean'},
                 log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
                 log_output: {type: 'array', items: {type: 'string'}},
                 log_directory: {type: 'string'},
@@ -172,16 +193,20 @@ const schema = {
                 baudrate: {type: 'number'},
                 rtscts: {type: 'boolean'},
                 soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: 'array', items: {type: 'number'}},
+                network_key: {type: ['array', 'string'], items: {type: 'number'}},
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
+                availability_blocklist: {type: 'array', items: {type: 'string'}},
+                availability_passlist: {type: 'array', items: {type: 'string'}},
+                // Deprecated, use block/passlist
                 availability_blacklist: {type: 'array', items: {type: 'string'}},
                 availability_whitelist: {type: 'array', items: {type: 'string'}},
                 report: {type: 'boolean'},
                 homeassistant_discovery_topic: {type: 'string'},
                 homeassistant_status_topic: {type: 'string'},
                 timestamp_format: {type: 'string'},
+                adapter_concurrent: {type: 'number'},
             },
         },
         map_options: {
@@ -262,6 +287,10 @@ const schema = {
             },
         },
     },
+    external_converters: {
+        type: 'array',
+        items: {type: 'string'},
+    },
     required: ['homeassistant', 'permit_join', 'mqtt'],
 };
 
@@ -275,13 +304,16 @@ function write() {
 
     // Read settings to check if we have to split devices/groups into separate file.
     const actual = yaml.read(file);
-    if (actual.mqtt && actual.mqtt.password && actual.mqtt.user) {
-        toWrite.mqtt.user = actual.mqtt.user;
-        toWrite.mqtt.password = actual.mqtt.password;
-    }
 
-    if (actual.advanced && actual.advanced.network_key) {
-        toWrite.advanced.network_key = actual.advanced.network_key;
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
     }
 
     if (typeof actual.devices === 'string') {
@@ -303,7 +335,12 @@ function write() {
 function validate() {
     const validate = ajv.compile(schema);
     const valid = validate(_settings);
-    const postfixes = utils.getPostfixes();
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    const postfixes = utils.getEndpointNames();
 
     // Verify that all friendly names are unique
     const names = [];
@@ -311,19 +348,37 @@ function validate() {
         if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
         if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
         if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (name.includes('#') || name.includes('+')) {
+            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+        }
         names.push(name);
     };
-    Object.values(_settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
-    Object.values(_settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
-    if (_settingsWithDefaults.mqtt.version !== 5) {
-        for (const device of Object.values(_settingsWithDefaults.devices)) {
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
             if (device.retention) {
                 throw new Error('MQTT retention requires protocol version 5');
             }
         }
     }
 
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
+
     return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
 
@@ -496,6 +551,7 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
+// Legacy: can be removed after bridgeLegacy has been removed
 function whitelistDevice(ID) {
     const settings = get();
     if (!settings.whitelist) {
@@ -510,6 +566,16 @@ function whitelistDevice(ID) {
     write();
 }
 
+function blockDevice(ID) {
+    const settings = get();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
 function banDevice(ID) {
     const settings = get();
     if (!settings.ban) {
@@ -587,6 +653,9 @@ function removeDeviceFromGroup(groupIDorName, keys) {
     const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
     const settings = get();
     const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
 
     const key = groupHasDevice(group, keys);
     if (key) {
@@ -602,16 +671,17 @@ function removeGroup(groupIDorName) {
     write();
 }
 
-function changeDeviceOptions(IDorName, newOptions) {
-    const device = getDeviceThrowIfNotExists(IDorName);
+function changeEntityOptions(IDorName, newOptions) {
     const settings = get();
-    const currentOptions = settings.devices[device.ID];
-
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
-        }
-    });
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
 
     write();
 }
@@ -643,13 +713,14 @@ module.exports = {
     getEntity,
     whitelistDevice,
     banDevice,
+    blockDevice,
     addDevice,
     removeDevice,
     addGroup,
     removeGroup,
     addDeviceToGroup,
     removeDeviceFromGroup,
-    changeDeviceOptions,
+    changeEntityOptions,
     changeFriendlyName,
 
     // For tests only

lib/util/utils.js

@@ -1,5 +1,7 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
+const humanizeDuration = require('humanize-duration');
+const data = require('./data');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -27,11 +29,15 @@ function toLocalISOString(dDate) {
         ':' + pad(tzOffset % 60);
 }
 
-const postfixes = [
+const endpointNames = [
     'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
     'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
     'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -40,10 +46,15 @@ function flatten(arr) {
     }, []);
 }
 
+function capitalize(s) {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
 const forceEndDevice = flatten(
     ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
         .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
+        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
+        .map((definition) => definition.zigbeeModel));
 
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
@@ -82,6 +93,10 @@ function formatDate(date, type) {
     case 'epoch':
         result = date;
         break;
+    case 'relative':
+        // https://github.com/EvanHahn/HumanizeDuration.js#options
+        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
+        break;
     default:
         throw new Error(`Unsupported type '${type}'`);
     }
@@ -119,13 +134,64 @@ function getObjectsProperty(objects, key, defaultValue) {
     return defaultValue;
 }
 
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+function parseJSON(value, failedReturnValue) {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+function* getExternalConvertersDefinitions(settings) {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converterModule = moduleName;
+
+        if (moduleName.endsWith('.js')) {
+            converterModule = data.joinPath(moduleName.split('.')[0]);
+        }
+
+        const converter = require(converterModule);
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+function toSnakeCase(value) {
+    if (typeof value === 'object') {
+        for (const key of Object.keys(value)) {
+            value[toSnakeCase(key)] = value[key];
+            delete value[key];
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
     objectHasProperties,
+    toSnakeCase,
     getObjectsProperty,
-    getPostfixes: () => postfixes,
+    getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
         return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
             (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
@@ -135,4 +201,8 @@ module.exports = {
     isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
+    getResponse,
+    capitalize,
+    parseJSON,
+    getExternalConvertersDefinitions,
 };

lib/util/yaml.js

@@ -1,6 +1,6 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
 
 function read(file) {
     try {
@@ -33,4 +33,12 @@ function writeIfChanged(file, content) {
     }
 }
 
-module.exports = {read, readIfExists, writeIfChanged};
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/zigbee.js

@@ -8,26 +8,9 @@ const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
-const postfixes = utils.getPostfixes();
+const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-const herdsmanSettings = {
-    network: {
-        panID: settings.get().advanced.pan_id,
-        extendedPanID: settings.get().advanced.ext_pan_id,
-        channelList: [settings.get().advanced.channel],
-        networkKey: settings.get().advanced.network_key,
-    },
-    databasePath: data.joinPath('database.db'),
-    databaseBackupPath: data.joinPath('database.db.backup'),
-    backupPath: data.joinPath('coordinator_backup.json'),
-    serialPort: {
-        baudRate: settings.get().advanced.baudrate,
-        rtscts: settings.get().advanced.rtscts,
-        path: settings.get().serial.port,
-    },
-};
-
 class Zigbee extends events.EventEmitter {
     constructor() {
         super();
@@ -36,10 +19,37 @@ class Zigbee extends events.EventEmitter {
 
     async start() {
         logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+            },
+        };
+
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
         logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
+        }
+
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
             this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
@@ -61,14 +71,16 @@ class Zigbee extends events.EventEmitter {
         logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
 
         for (const device of this.getClients()) {
-            // If a whitelist is used, all other device will be removed from the network.
-            if (settings.get().whitelist.length > 0) {
-                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
                     device.removeFromNetwork();
                 }
-            } else if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
                 device.removeFromNetwork();
             }
         }
@@ -127,10 +139,22 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDevicesByType(type);
     }
 
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
+     */
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
         );
 
         if (typeof key === 'string' || typeof key === 'number') {
@@ -149,13 +173,13 @@ class Zigbee extends events.EventEmitter {
                 };
             }
 
-            let postfix = postfixes.find((p) => key.endsWith(`/${p}`));
-            const postfixByNumber = key.match(keyEndpointByNumber);
-            if (!postfix && postfixByNumber) {
-                postfix = Number(postfixByNumber[1]);
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
             }
-            if (postfix) {
-                key = key.replace(`/${postfix}`, '');
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
             }
 
             const entity = settings.getEntity(key);
@@ -167,19 +191,17 @@ class Zigbee extends events.EventEmitter {
                     return null;
                 }
 
-                const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
-                const endpoints = mapped && mapped.endpoint ? mapped.endpoint(device) : null;
-                let isDefaultEndpoint = true;
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
-                if (postfix) {
-                    isDefaultEndpoint = false;
-                    if (postfixByNumber) {
-                        endpoint = device.getEndpoint(postfix);
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
                     } else {
-                        assert(mapped != null, `Postfix '${postfix}' is given but device is unsupported`);
-                        assert(endpoints != null, `Postfix '${postfix}' is given but device defines no endpoints`);
-                        const endpointID = endpoints[postfix];
-                        assert(endpointID, `Postfix '${postfix}' is given but device has no such endpoint`);
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
                         endpoint = device.getEndpoint(endpointID);
                     }
                 } else if (endpoints && endpoints['default']) {
@@ -188,24 +210,31 @@ class Zigbee extends events.EventEmitter {
                     endpoint = device.endpoints[0];
                 }
 
-                const endpointName = endpoints ? Object.entries(endpoints).find((e) => e[1] === endpoint.ID)[0] : null;
                 return {
-                    type: 'device', device, settings: entity, mapped, endpoint, name: entity.friendlyName,
-                    isDefaultEndpoint, endpointName,
+                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
                 return {type: 'group', group, settings: entity, name: entity.friendlyName};
             }
-        } else {
+        } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
             return {
                 type: 'device',
                 device: key,
+                endpoint: key.endpoints[0],
                 settings: setting,
-                mapped: zigbeeHerdsmanConverters.findByZigbeeModel(key.modelID),
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting ? setting.friendlyName : key.groupID,
             };
         }
     }
@@ -223,21 +252,23 @@ class Zigbee extends events.EventEmitter {
     }
 
     acceptJoiningDeviceHandler(ieeeAddr) {
-        // If set whitelist devices, all other device will be rejected to join the network
-        if (settings.get().whitelist.length > 0) {
-            if (settings.get().whitelist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
                 return true;
             } else {
-                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
                 return false;
             }
-        } else if (settings.get().ban.length > 0) {
-            if (settings.get().ban.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
                 return false;
             } else {
-                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
                 return true;
             }
         } else {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.12.0",
+  "version": "1.14.2",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13"
+    "node": "^10 || ^12 || ^13 || ^14"
   },
   "keywords": [
     "xiaomi",
@@ -33,20 +33,22 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "*",
-    "debounce": "*",
-    "fast-deep-equal": "*",
-    "git-last-commit": "*",
-    "js-yaml": "*",
-    "mkdir-recursive": "*",
-    "moment": "*",
-    "mqtt": "*",
-    "object-assign-deep": "*",
-    "rimraf": "*",
-    "semver": "*",
-    "winston": "*",
-    "zigbee-herdsman": "0.12.64",
-    "zigbee-herdsman-converters": "12.0.51"
+    "ajv": "^6.12.3",
+    "debounce": "^1.2.0",
+    "fast-deep-equal": "^3.1.3",
+    "git-last-commit": "^1.0.0",
+    "humanize-duration": "^3.23.1",
+    "js-yaml": "^3.14.0",
+    "mkdir-recursive": "^0.4.0",
+    "moment": "^2.27.0",
+    "mqtt": "^4.1.0",
+    "object-assign-deep": "^0.4.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.2",
+    "winston": "^3.3.3",
+    "winston-syslog": "^2.4.4",
+    "zigbee-herdsman": "0.12.108",
+    "zigbee-herdsman-converters": "12.0.147"
   },
   "devDependencies": {
     "eslint": "*",

test/assets/exampleExtension.js

@@ -0,0 +1,11 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/assets/mock-external-converter-multiple.js

@@ -0,0 +1,20 @@
+const homeassistantSwitch = {
+    type: 'switch',
+    object_id: 'switch',
+    discovery_payload: {
+        payload_off: 'OFF',
+        payload_on: 'ON',
+        value_template: '{{ value_json.state }}',
+        command_topic: true,
+    },
+};
+
+const mockDevices = [{
+    mock: 1,
+    model: 'external_converters_device',
+    homeassistant: [homeassistantSwitch],
+}, {
+    mock: 2
+}];
+
+module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -0,0 +1,5 @@
+const mockDevice = {
+    mock: true
+};
+
+module.exports = mockDevice;

test/availability.test.js

@@ -15,11 +15,11 @@ const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device availability', () => {
+describe('Availability', () => {
     let controller;
 
     function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'DeviceAvailability');
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
     beforeEach(async () => {
@@ -197,7 +197,21 @@ describe('Device availability', () => {
         expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
     });
 
-    it('Should not ping devices blacklisted by friendly name', async () => {
+    it('Should not ping devices on blocklist by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
         await controller.stop();
@@ -211,9 +225,9 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not ping devices blacklisted by IEEE address', async () => {
+    it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -238,7 +252,21 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
         await controller.stop();
@@ -252,9 +280,9 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -266,10 +294,10 @@ describe('Device availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         getExtension().state[device.ieeeAddr] = false;
-        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
+        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -352,4 +380,21 @@ describe('Device availability', () => {
 
         device.lastSeen = defaultLastSeen;
     });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
 });

test/bind.test.js

@@ -0,0 +1,406 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+describe('Bind', () => {
+    let controller;
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind = jest.fn();
+            endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        MQTT.publish.mockClear();
+    });
+
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
+    });
+
+    it('Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'button'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: 'Coordinator'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: '1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementation(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', JSON.stringify({from: 'remote', to: target}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            JSON.stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when source not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when target not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', JSON.stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            JSON.stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
+    it('Legacy api: Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'Coordinator');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'group_1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', '1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+
+    it('Legacy api: Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch_double/right');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Legacy api: Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Legacy api: Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
+    });
+});

test/bridge.test.js

@@ -0,0 +1,605 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+
+describe('Bridge', () => {
+    let controller;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        settings.set(['advanced', 'legacy_api'], false);
+        settings.set(['experimental', 'new_api'], true);
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        device.removeFromDatabase.mockClear();
+        device.removeFromNetwork.mockClear();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+    });
+
+    it('Should publish bridge info on startup', async () => {
+        const version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/info',
+          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"log_level":"info","permit_join":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          JSON.stringify([{"ieee_address":"0x000b57fffec6a5b2","type":"Router","network_address":40369,"supported":true,"friendly_name":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"power_source":"Mains (single phase)","date_code":null,"interviewing":false,"interview_completed":true},{"ieee_address":"0x0017880104e45518","type":"EndDevice","network_address":6536,"supported":false,"friendly_name":"0x0017880104e45518","definition":null,"power_source":"Battery","date_code":null,"interviewing":false,"interview_completed":true}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/groups',
+          JSON.stringify([{"id":1,"friendly_name":"group_1","members":[]},{"id":15071,"friendly_name":"group_tradfri_remote","members":[]},{"id":99,"friendly_name":99,"members":[]},{"id":11,"friendly_name":"group_with_tradfri","members":[]},{"id":2,"friendly_name":"group_2","members":[]}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device joined', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_joined","data":{"friendly_name":"bulb","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device interview started', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'started'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"started","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview failed', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'failed'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"failed","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview successful', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'successful'});
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.unsupported, status: 'successful'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"successful","ieee_address":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+          'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"device_interview","data":{"friendly_name":"0x0017880104e45518","status":"successful","ieee_address":"0x0017880104e45518","supported":false,"definition":null}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device leaves', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: zigbeeHerdsman.devices.bulb.ieeeAddr});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"device_leave","data":{"ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put transaction in response when request is done with transaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false, "transaction": 22}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when request fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when format is incorrect', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', JSON.stringify({"value_not_good": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            JSON.stringify({"data":{},"status":"error","error": "No value given"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Coverage satisfaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        const device = zigbeeHerdsman.devices.bulb;
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+    });
+
+    it('Should allow a healthcheck', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/health_check', '');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/health_check',
+            JSON.stringify({"data":{"healthy": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to remove device by string', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb');
+        await flushPromises();
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual([]);
+    });
+
+    it('Should allow to remove device by object ID', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb"}));
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb", force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(device.removeFromNetwork).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":false,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to block device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb", block: true, force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{"id": "bulb","block":true,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual(["0x000b57fffec6a5b2"]);
+    });
+
+    it('Should allow to remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', 'group_1');
+        await flushPromises();
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            JSON.stringify({"data":{"id": "group_1", "force": false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', JSON.stringify({id: "group_1", force: true}));
+        await flushPromises();
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            JSON.stringify({"data":{"id": "group_1", "force": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on removing non-existing device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "non-existing-device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{},"status":"error","error":"Device 'non-existing-device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when remove device fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('device timeout')})
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', JSON.stringify({id: "bulb"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            JSON.stringify({"data":{},"status":"error","error":"Failed to remove device 'bulb' (block: false, force: false) (Error: device timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename group', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', JSON.stringify({from: 'group_1', to: 'group_new_name'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(settings.getGroup('group_new_name')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_new_name", "friendlyName": "group_new_name", "optimistic": true, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/rename',
+            JSON.stringify({"data":{"from":"group_1","to":"group_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device rename payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from_bla: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on non-existing device rename', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({from: 'bulb_not_existing', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"Device 'bulb_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when renaming last joined device but none has joined', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', JSON.stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            JSON.stringify({"data":{},"status":"error","error":"No device has joined since start"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change device options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', JSON.stringify({options: {retain: false, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": false, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            JSON.stringify({"data":{"from":{"retain": true},"to":{"retain": false,"transition":1}, "id":"bulb"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change group options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": false, "friendlyName": "group_1", "optimistic": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/options', JSON.stringify({options: {retain: true, transition: 1}, id: 'group_1'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": true, "friendlyName": "group_1", "optimistic": true, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/options',
+            JSON.stringify({"data":{"from":{"optimistic": true,"retain": false},"to":{"optimistic": true,"retain": true,"transition":1}, "id":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device change options payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', JSON.stringify({options_: {retain: true, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group by string', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', 'group_193');
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 3, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{"friendly_name":"group_193","id": 3},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group with ID', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', JSON.stringify({friendly_name: "group_193", id: 9}));
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 9, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{"friendly_name":"group_193","id": 9},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when add with invalid payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', JSON.stringify({friendly_name9: "group_193"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            JSON.stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            JSON.stringify({"data":{"value":"ISO_8601"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'invalid_one');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            JSON.stringify({"data":{},"status":"error","error":"'invalid_one' is not an allowed value, allowed: disable,ISO_8601,epoch,ISO_8601_local"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'not_valid');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            JSON.stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: true,false"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set log level', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            JSON.stringify({"data":{"value":'debug'},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set log level when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'not_valid');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            JSON.stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: error,warn,info,debug"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (succeeds)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            JSON.stringify({"data":{},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (fails)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            JSON.stringify({"data":{},"status":"error","error":"Failed to factory reset device through Touchlink"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/configure.test.js

@@ -9,7 +9,7 @@ const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device configure', () => {
+describe('Configure', () => {
     let controller;
 
     expectRemoteConfigured = () => {
@@ -90,8 +90,52 @@ describe('Device configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Should allow to reconfigure manually', async () => {
+    it('Should allow to configure via MQTT', async () => {
         expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "remote"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "not_existing_device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when configure fails', async () => {
+        zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "remote"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device has no configure', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', JSON.stringify({id: "bulb", transaction: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should allow to reconfigure manually', async () => {
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
@@ -99,13 +143,13 @@ describe('Device configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Shouldnt manually reconfigure when device does not exist', async () => {
+    it('Legacy api: Shouldnt manually reconfigure when device does not exist', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
         await flushPromises();
         expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
     });
 
-    it('Should skip reconfigure when device does not require this', async () => {
+    it('Legacy api: Should skip reconfigure when device does not require this', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
         await flushPromises();
         expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
@@ -153,7 +197,7 @@ describe('Device configure', () => {
     });
 
     it('Should configure max 3 times when fails', async () => {
-        controller.extensions.find((e) => e.constructor.name === 'DeviceConfigure').attempts = {};
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
         const device = zigbeeHerdsman.devices.remote;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);

test/controller.test.js

@@ -30,7 +30,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),"serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -46,6 +46,13 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
     });
 
+    it('Start controller when permit join fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error("failed!")});
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+    });
+
     it('Start controller with specific MQTT settings', async () => {
         const ca = tmp.fileSync().name;
         fs.writeFileSync(ca, "ca");
@@ -85,6 +92,14 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should generate network_key when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+    });
+
     it('Start controller should publish cached states', async () => {
         data.writeDefaultState();
         await controller.start();
@@ -93,6 +108,15 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
     });
 
+    it('Start controller should not publish cached states when disabled', async () => {
+        settings.set(['advanced', 'cache_state_send_on_startup'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        const publishedTopics = MQTT.publish.mock.calls.map(m => m[0]);
+        expect(publishedTopics).toEqual(expect.not.arrayContaining(["zigbee2mqtt/bulb", "zigbee2mqtt/remote"]));
+    });
+
     it('Start controller should not publish cached states when cache_state is false', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeDefaultState();
@@ -135,6 +159,14 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
+    it('Should remove device not on passlist on startup', async () => {
+        settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
     it('Should remove non whitelisted devices on startup', async () => {
         settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -143,6 +175,14 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
     });
 
+    it('Should remove device on blocklist on startup', async () => {
+        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
     it('Should remove banned devices on startup', async () => {
         settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -242,9 +282,16 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
     });
 
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
     it('On zigbee event message from unkown device should create it', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
         expect(settings.getDevice(device.ieeeAddr)).toBeNull();
         const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
         await zigbeeHerdsman.events.message(payload);
@@ -261,48 +308,48 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('acceptJoiningDeviceHandler reject banned device', async () => {
+    it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+    it('acceptJoiningDeviceHandler accept device not on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], ['123']);
+        settings.set(['blocklist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler accept device on passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler reject device not in passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], ['123']);
+        settings.set(['passlist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+    it('acceptJoiningDeviceHandler should prefer passlist above blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+    it('acceptJoiningDeviceHandler accept when not on blocklist and passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
@@ -495,6 +542,21 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should allow to disable state persistency', async () => {
+        settings.set(['advanced', 'cache_state_persistent'], false);
+        data.removeState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", JSON.stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        await controller.stop();
+        expect(data.stateExists()).toBeFalsy();
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();
@@ -522,4 +584,15 @@ describe('Controller', () => {
         await flushPromises();
         expect(controller.state.state).toStrictEqual({});
     });
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
 });

test/deviceBind.test.js

@@ -1,192 +0,0 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const MQTT = require('./stub/mqtt');
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-
-describe('Device bind', () => {
-    let controller;
-
-    mockClear = (device) => {
-        for (const endpoint of device.endpoints) {
-            endpoint.read.mockClear();
-            endpoint.write.mockClear();
-            endpoint.configureReporting.mockClear();
-            endpoint.bind.mockClear();
-            endpoint.unbind.mockClear();
-        }
-    }
-
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
-        MQTT.publish.mockClear();
-    });
-
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
-    });
-
-    it('Should bind', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
-        await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
-        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
-    });
-
-    it('Should unbind', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'bulb_color');
-        await flushPromises();
-        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
-    });
-
-    it('Should unbind coordinator', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        endpoint.unbind.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'Coordinator');
-        await flushPromises();
-        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
-    });
-
-    it('Should bind to groups', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.groups.group_1;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'group_1');
-        await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
-        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
-    });
-
-    it('Should bind to group by number', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.groups.group_1;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', '1');
-        await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
-        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
-        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
-    });
-
-    it('Should log when bind fails', async () => {
-        logger.error.mockClear();
-        const device = zigbeeHerdsman.devices.remote;
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
-        await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
-    });
-
-    it('Should bind from non default endpoints', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
-        const endpoint = device.getEndpoint(2);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch_double/right');
-        await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
-    });
-
-    it('Should bind to default endpoint returned by endpoints()', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
-        const endpoint = device.getEndpoint(2);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch');
-        await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
-    });
-
-    it('Should unbind from default_bind_group', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const target = 'default_bind_group';
-        const endpoint = device.getEndpoint(1);
-        mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
-        await flushPromises();
-        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
-        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
-        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
-    });
-});

test/externalConverters.test.js

@@ -0,0 +1,115 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const path = require('path');
+const fs = require('fs');
+
+zigbeeHerdsmanConverters.addDeviceDefinition = jest.fn();
+
+const mocksClear = [zigbeeHerdsmanConverters.addDeviceDefinition, zigbeeHerdsman.permitJoin,
+    mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+jest.mock(
+    'mock-external-converter-module', () => {
+        return {
+            mock: true
+        };
+    }, {
+        virtual: true
+    });
+
+jest.mock(
+    'mock-multiple-external-converter-module', () => {
+        return [{
+            mock: 1
+        }, {
+            mock: 2
+        }];
+    }, {
+        virtual: true
+    });
+
+describe('Loads external converters', () => {
+    let controller;
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        await flushPromises();
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        data.writeEmptyState();
+        settings._reRead();
+    });
+
+    it('Does not load external converters', async () => {
+        settings.set(['external_converters'], []);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
+    });
+
+    it('Loads external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1,
+            model: 'external_converters_device',
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+
+    it('Loads external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+});

test/group.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -15,11 +16,13 @@ describe('Groups', () => {
 
     beforeEach(() => {
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
         settings._reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     })
 
     it('Apply group updates add', async () => {
@@ -79,7 +82,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
-    it('Add to group via MQTT', async () => {
+    it('Legacy api: Add to group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -94,7 +97,7 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Add to group with slashes via MQTT', async () => {
+    it('Legacy api: Add to group with slashes via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups["group/with/slashes"];
@@ -115,7 +118,7 @@ describe('Groups', () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
-    it('Add to group via MQTT with postfix', async () => {
+    it('Legacy api: Add to group via MQTT with postfix', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -128,7 +131,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+    it('Legacy api: Add to group via MQTT with postfix shouldnt add it twice', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -143,7 +146,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove from group via MQTT', async () => {
+    it('Legacy api: Remove from group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -158,7 +161,7 @@ describe('Groups', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+    it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -172,7 +175,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
     });
 
-    it('Remove from group via MQTT with postfix variant 1', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 1', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -186,7 +189,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 2', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 2', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -200,7 +203,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 3', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 3', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -214,7 +217,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group all', async () => {
+    it('Legacy api: Remove from group all', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -243,7 +246,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Log when adding to non-existing group', async () => {
+    it('Legacy api: Log when adding to non-existing group', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -252,7 +255,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
     });
 
-    it('Log when adding to non-existing device', async () => {
+    it('Legacy api: Log when adding to non-existing device', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -375,7 +378,7 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -400,8 +403,8 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -426,9 +429,9 @@ describe('Groups', () => {
         await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF"}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -454,7 +457,269 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', JSON.stringify({device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove_all',
+            JSON.stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', JSON.stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            JSON.stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', JSON.stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            JSON.stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/homeassistant.test.js

@@ -5,6 +5,9 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
+const fs = require('fs');
+const path = require('path');
+const HomeAssistant = require('../lib/extension/homeassistant');
 
 describe('HomeAssistant extension', () => {
     beforeEach(async () => {
@@ -20,7 +23,6 @@ describe('HomeAssistant extension', () => {
 
     it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
         const missing = [];
-        const HomeAssistant = require('../lib/extension/homeassistant');
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
@@ -32,6 +34,26 @@ describe('HomeAssistant extension', () => {
         expect(missing).toHaveLength(0);
     });
 
+    it('Should not have duplicate type/object_ids in a mapping', () => {
+        const duplicated = [];
+        const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
+
+        require('zigbee-herdsman-converters').devices.forEach((d) => {
+            const mapping = ha._getMapping()[d.model];
+            const cfg_type_object_ids = [];
+
+            mapping.forEach((c) => {
+                if (cfg_type_object_ids.includes(c['type'] + '/' + c['object_id'])) {
+                    duplicated.push(d.model);
+                } else {
+                    cfg_type_object_ids.push(c['type'] + '/' + c['object_id']);
+                }
+            });
+        });
+
+        expect(duplicated).toHaveLength(0);
+    });
+
     it('Should discover devices', async () => {
         controller = new Controller(false);
         await controller.start();
@@ -183,7 +205,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '°C',
             'device_class': 'temperature',
-            'value_template': "{{ (value_json.temperature | float) | round(1) }}",
+            'value_template': "{{ value_json.temperature }}",
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_temperature',
@@ -208,7 +230,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': '%',
             'device_class': 'humidity',
-            'value_template': '{{ (value_json.humidity | float) | round(0) }}',
+            'value_template': '{{ value_json.humidity }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_humidity',
@@ -233,7 +255,7 @@ describe('HomeAssistant extension', () => {
         payload = {
             'unit_of_measurement': 'hPa',
             'device_class': 'pressure',
-            'value_template': '{{ (value_json.pressure | float) | round(2) }}',
+            'value_template': '{{ value_json.pressure }}',
             'state_topic': 'zigbee2mqtt/weather_sensor',
             'json_attributes_topic': 'zigbee2mqtt/weather_sensor',
             'name': 'weather_sensor_pressure',
@@ -496,7 +518,7 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        // 1 publish is the publish from deviceReceive
+        // 1 publish is the publish from receive
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
@@ -537,7 +559,7 @@ describe('HomeAssistant extension', () => {
         );
     });
 
-    it('Shouldnt discover when message has no device yet', async () => {
+    it('Shouldnt discover when device leaves', async () => {
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -551,12 +573,39 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
+    it('Should send all status when home assistant comes online (default topic)', async () => {
+        jest.useFakeTimers();
+        data.writeDefaultState();
+        controller = new Controller();
+        await controller.start();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
+        await flushPromises();
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/status', 'online');
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote',
+            '{"brightness":255,"update_available":false}',
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
     it('Should send all status when home assistant comes online', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
         controller = new Controller();
         await controller.start();
         await flushPromises();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
         MQTT.publish.mockClear();
         await MQTT.events.message('hass/status', 'online');
         await flushPromises();
@@ -564,13 +613,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',
+            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255}',
+            '{"brightness":255,"update_available":false}',
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -918,4 +967,49 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
+
+    it('Should counter an action/click payload with an empty payload', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
+        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
+    });
+
+    it('Load Home Assistant mapping from external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const beforeCount = Object.entries((new HomeAssistant(null, null, null, null, {on: () => {}}))._getMapping()).length;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
+        await controller.start();
+        await flushPromises();
+        const afterCount = Object.entries(ha._getMapping()).length;
+        expect(beforeCount + 1).toStrictEqual(afterCount);
+
+        const homeassistantSwitch = {
+            type: 'switch',
+            object_id: 'switch',
+            discovery_payload: {
+                payload_off: 'OFF',
+                payload_on: 'ON',
+                value_template: '{{ value_json.state }}',
+                command_topic: true,
+            },
+        };
+        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+    });
 });

test/legacy/bridgeLegacy.test.js

@@ -1,19 +1,19 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-const MQTT = require('./stub/mqtt');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 
 
-describe('Bridge config', () => {
+describe('Bridge legacy', () => {
     let controller;
 
     beforeAll(async () => {
-        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
         controller = new Controller();
         await controller.start();
     })
@@ -100,6 +100,21 @@ describe('Bridge config', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
     });
 
     it('Should allow permit join', async () => {
@@ -161,8 +176,8 @@ describe('Bridge config', () => {
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
-        expect(payload[0]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[1]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 
@@ -303,6 +318,20 @@ describe('Bridge config', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
         await flushPromises();
         expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
         expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',

test/logger.test.js

@@ -98,4 +98,14 @@ describe('Logger', () => {
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
     });
+
+    it('Should allow to disable log rotation', () => {
+        settings.set(['advanced', 'log_rotation'], false);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].maxFiles).toBeNull();
+        expect(pipes[1].tailable).toBeFalsy();
+        expect(pipes[1].maxsize).toBeNull();
+    });
 });

test/networkMap.test.js

@@ -26,6 +26,7 @@ describe('Networkmap', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
@@ -75,7 +76,7 @@ describe('Networkmap', () => {
         unsupported_router.routingTable = () => {throw new Error('failed')};
     }
 
-    it('Output raw networkmap', async () => {
+    it('Output raw networkmap legacy api', async () => {
         mock();
         MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
         await flushPromises();
@@ -126,11 +127,7 @@ describe('Networkmap', () => {
         expect(JSON.parse(call[1])).toStrictEqual(expected);
     });
 
-    it('Output graphviz networkmap', async () => {
-        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
-        const getHours = Date.prototype.getHours;
-        Date.prototype.getTimezoneOffset = () => -60;
-        Date.prototype.getHours = () => 1;
+    it('Output graphviz networkmap legacy api', async () => {
         mock();
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = null;
@@ -146,16 +143,16 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
               "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
               "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
             }`;
 
@@ -165,7 +162,149 @@ describe('Networkmap', () => {
         for (let i = 0; i < expectedLines.length; i++) {
             expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
         }
-        Date.prototype.getTimezoneOffset = getTimezoneOffset;
-        Date.prototype.getHours = getHours;
+    });
+
+    it('Output plantuml networkmap legacy api', async () => {
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'plantuml');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/plantuml');
+
+        const expected = `' paste into: https://www.planttext.com/
+
+        @startuml
+        card 0x0017880104e45525 [
+        0x0017880104e45525
+        ---
+        0x0017880104e45525 (6536) failed: lqi,routingTable
+        ---
+        Boef notSupportedModelID
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b2 [
+        bulb
+        ---
+        0x000b57fffec6a5b2 (40369)
+        ---
+        IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b3 [
+        bulb_color
+        ---
+        0x000b57fffec6a5b3 (40399)
+        ---
+        Philips Hue Go (7146060PH)
+        ---
+        unknown
+        ]
+
+        card 0x0017880104e45521 [
+        button_double_key
+        ---
+        0x0017880104e45521 (6538)
+        ---
+        Xiaomi Aqara double key wireless wall switch (WXKG02LM)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x0017880104e45559 [
+        cc2530_router
+        ---
+        0x0017880104e45559 (6540)
+        ---
+        Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x00124b00120144ae [
+        Coordinator
+        ---
+        0x00124b00120144ae (0)
+        ---
+        0 seconds ago
+        ]
+
+        0x000b57fffec6a5b3 --> 0x00124b00120144ae: 120
+        0x000b57fffec6a5b2 --> 0x00124b00120144ae: 92
+        0x000b57fffec6a5b3 --> 0x000b57fffec6a5b2: 110
+        0x0017880104e45559 --> 0x000b57fffec6a5b2: 100
+        0x0017880104e45521 --> 0x0017880104e45559: 130
+
+        @enduml`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+    });
+
+    it('Should output raw networkmap', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', JSON.stringify({type: 'raw', routes: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
+
+        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
+
+        const expected = {
+            status: 'ok',
+            data: {
+                type: 'raw',
+                routes: true,
+                value: {
+                    "nodes":[
+                        {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
+                        {"lastSeen": null,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
+                        {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
+                        {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
+                    ],
+                    "links":[
+                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
+                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
+                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
+                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
+                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
+                    ]
+                }
+            }
+        };
+
+        const actual = JSON.parse(call[1]);
+        actual.data.value = JSON.parse(actual.data.value);
+        expect(actual).toStrictEqual(expected);
+    });
+
+    it('Should throw error when rquesting invalid type', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', 'not_existing');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/networkmap',
+            JSON.stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/onEvent.test.js

@@ -13,11 +13,11 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const mockOnEvent = jest.fn();
-const mappedLivolo = zigbeeHerdsmanConverters.findByZigbeeModel(zigbeeHerdsman.devices.LIVOLO.modelID);
+const mappedLivolo = zigbeeHerdsmanConverters.findByDevice(zigbeeHerdsman.devices.LIVOLO);
 mappedLivolo.onEvent = mockOnEvent;
 zigbeeHerdsmanConverters.onEvent = jest.fn();
 
-describe('Device event', () => {
+describe('On event', () => {
     let controller;
     const device = zigbeeHerdsman.devices.LIVOLO;
 

test/otaUpdate.test.js

@@ -18,6 +18,7 @@ describe('OTA update', () => {
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings._reRead();
+        settings.set(['experimental', 'new_api'], true);
         data.writeEmptyState();
         controller = new Controller();
         await controller.start();
@@ -28,6 +29,8 @@ describe('OTA update', () => {
     it('Should subscribe to topics', async () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
     });
 
     it('Should OTA update a device', async () => {
@@ -38,134 +41,164 @@ describe('OTA update', () => {
             count++;
             return {swBuildId: count, dateCode: '2019010' + count}
         });
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             onUpdate(0, null);
             onUpdate(10, 3600);
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', 'bulb');
         await flushPromises();
         expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
-        expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
         expect(device.softwareBuildID).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle when OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             throw new Error('Update failed');
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', JSON.stringify({id: "bulb"}));
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should be able to check if OTA update is available', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
 
-        logger.info.mockClear();
+        MQTT.publish.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.error.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail when device does not exist', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "not_existing_deviceooo");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "ZNLDP12LM");
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should refuse to check/update when already in progress', async () => {
         jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
             return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
         });
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
         jest.runAllTimers();
         await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            JSON.stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
-    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+    it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         let count = 0;
-        endpoint.read.mockImplementation(() => {
-            if (count === 1) throw new Error('Failed!')
-            count++;
-            return {swBuildId: 1, dateCode: '2019010'}
-        });
+        endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', "bulb");
         await flushPromises();
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            JSON.stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should check for update when device requests it', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const data = {imageType: 12382};
-        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
         const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
@@ -201,4 +234,127 @@ describe('OTA update', () => {
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
     });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+
+    it('Legacy api: Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Legacy api: Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Legacy api: Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Legacy api: Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Legacy api: Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Legacy api: Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
 });

test/publish.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
@@ -21,7 +22,7 @@ const expectNothingPublished = () => {
     })
 }
 
-describe('Entity publish', () => {
+describe('Publish', () => {
     let controller;
 
     beforeAll(async () => {
@@ -48,6 +49,8 @@ describe('Entity publish', () => {
         Object.values(zigbeeHerdsman.groups).forEach((g) => {
             g.command.mockClear();
         });
+
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
     });
 
     it('Should publish messages to zigbee devices', async () => {
@@ -249,14 +252,14 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {x: 100, y: 50}, brightness: 20, state: 'OFF'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(2);
-        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(1, "lightingColorCtrl", "moveToColor", {colorx: 6553500, colory: 3276750, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenNthCalledWith(2, "genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({color: {x: 100, y: 50}, color_temp: 62, state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -266,10 +269,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {r: 100, g: 200, b: 10}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -278,10 +281,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({color: {rgb: '100,200,10'}}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should publish messages to zigbee devices with color rgb', async () => {
@@ -337,7 +340,7 @@ describe('Entity publish', () => {
         expect(group.command).toHaveBeenCalledWith("genOnOff", "off", {}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/group_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('Should publish messages to groups color', async () => {
@@ -411,6 +414,31 @@ describe('Entity publish', () => {
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
     });
 
+    it('Should handle get with multiple endpoints', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_left: '', state_right: ''}));
+        await flushPromises();
+        expect(endpoint2.read).toHaveBeenCalledTimes(1);
+        expect(endpoint2.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should log error when device has no such endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint2 = device.getEndpoint(2);
+        const endpoint3 = device.getEndpoint(3);
+        logger.error.mockClear();
+        await MQTT.events.message('zigbee2mqtt/0x0017880104e45542/get', JSON.stringify({state_center: '', state_right: ''}));
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'wall_switch_double' has no endpoint 'center'`);
+        expect(endpoint2.read).toHaveBeenCalledTimes(0);
+        expect(endpoint3.read).toHaveBeenCalledTimes(1);
+        expect(endpoint3.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
     it('Should not respond to bridge/config/devices/get', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', JSON.stringify({state: 'ON'}));
         await flushPromises();
@@ -475,10 +503,10 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set/color', '#64C80A');
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 17806, colory: 43155, transtime: 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("lightingColorCtrl", "moveToColor", {colorx: 16764, colory: 40979, transtime: 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 156, color: {x: 0.2717, y: 0.6585}});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({color_temp: 162, color: {x: 0.2558, y: 0.6253}});
     });
 
     it('Should parse set with ieeeAddr topic', async () => {
@@ -496,6 +524,21 @@ describe('Entity publish', () => {
         expectNothingPublished();
     });
 
+    it('Should send state update on toggle specific endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(2);
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'ON');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/wall_switch_double/left/set', 'TOGGLE');
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "toggle", {}, {});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'ON'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/wall_switch_double", JSON.stringify({state_left: 'OFF'}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Should parse set with postfix topic and attribute', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(2);
@@ -575,7 +618,7 @@ describe('Entity publish', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON', brightness: 100});
     });
 
-    it('Should use transition when brightness', async () => {
+    it('Should use transition on brightness command', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
         const endpoint = device.getEndpoint(1);
@@ -586,6 +629,56 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
     });
 
+    it('Should use transition from device_options on brightness command', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['device_options'], {transition: 20});
+        const endpoint = device.getEndpoint(1);
+        const payload = {brightness: 20};
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 20, transtime: 200}, {}]);
+    });
+
+    it('Should turn bulb on with full brightness when transition is used and no brightness is known', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3799
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "OFF", "transition": 0.5}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 0.5}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color',
+            JSON.stringify({state: 'OFF', brightness: 0}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenNthCalledWith(2,
+            'zigbee2mqtt/bulb_color',
+            JSON.stringify({state: 'ON', brightness: 254}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 5}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 5}, {}]);
+    });
+
+    it('Transition parameter should not influence brightness on state ON', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3563
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 50}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON"}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 10}, {}]);
+    });
+
     it('Should use transition when color temp', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['devices', device.ieeeAddr, 'transition'], 20);
@@ -642,11 +735,20 @@ describe('Entity publish', () => {
     it('Should turn device off when brightness 0 is send', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        const payload = {'brightness': 0};
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 50, state: 'ON'}));
         await flushPromises();
-        expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "off", {}, {}]);
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 0}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 50, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 50}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
     it('Should turn device off when brightness 0 is send with light_brightness converter', async () => {
@@ -666,7 +768,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 45510.416666666664, "saturation": 127, "transtime": 0,}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["lightingColorCtrl", "enhancedMoveToHueAndSaturation", {"direction": 0, "enhancehue": 44891.475, "saturation": 199.21474, "transtime": 0,}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"color":{"hue":250,"saturation":50}});
@@ -712,7 +814,6 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("closuresWindowCovering", "stop", {}, {});
     });
 
-
     it('Should turn device on with on/off when transition is provided', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -720,7 +821,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 30}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 30}, {}]);
     });
 
     it('Should turn device on with on/off with transition when transition 0 is provided', async () => {
@@ -730,7 +831,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify(payload));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 255, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 254, transtime: 0}, {}]);
     });
 
     it('Should turn device off with onOff on off with transition', async () => {
@@ -743,7 +844,7 @@ describe('Entity publish', () => {
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 10}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
     });
 
     it('When device is turned off and on with transition with report enabled it should restore correct brightness', async () => {
@@ -761,7 +862,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(1);
         expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
 
         // Bulb reports brightness while decreasing brightness
         await zigbeeHerdsman.events.message({data: {currentLevel: 1}, cluster: 'genLevelCtrl', device, endpoint, type: 'attributeReport', linkquality: 10});
@@ -778,6 +879,32 @@ describe('Entity publish', () => {
         expect(MQTT.publish.mock.calls[2]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200, linkquality: 10}), {"qos": 0, "retain": false}, expect.any(Function)]);
     });
 
+    it('When device is turned off with transition and turned on WITHOUT transition it should restore the brightness', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        // Set initial brightness in state
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({brightness: 200}));
+        await flushPromises();
+        endpoint.command.mockClear();
+        MQTT.publish.mockClear();
+
+        // Turn off
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF', transition: 3}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 0, transtime: 30}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'OFF', brightness: 0}), {"qos": 0, "retain": false}, expect.any(Function)]);
+
+        // Turn on again
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 200, transtime: 0}, {}]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[1]).toEqual(["zigbee2mqtt/bulb_color", JSON.stringify({state: 'ON', brightness: 200}), {"qos": 0, "retain": false}, expect.any(Function)]);
+    });
+
     it('Home Assistant: should set state', async () => {
         settings.set(['homeassistant'], true);
         const device = zigbeeHerdsman.devices.bulb_color;
@@ -860,7 +987,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bulb_color');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', brightness: 0});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -873,7 +1000,7 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledWith("genOnOff", "off", {}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/led_controller_1');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'OFF', brightness: 0});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -952,7 +1079,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/roller_shutter/set', JSON.stringify({state: 'OPEN'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevel", {"level": "255", "transtime": 0}, {});
+        expect(endpoint.command).toHaveBeenCalledWith("genLevelCtrl", "moveToLevelWithOnOff", {"level": "255", "transtime": 0}, {});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/roller_shutter');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({position: 100});
@@ -965,7 +1092,7 @@ describe('Entity publish', () => {
         await MQTT.events.message('zigbee2mqtt/MKS-CM-W5/l3/set', JSON.stringify({state: 'ON'}));
         await flushPromises();
         expect(endpoint.command).toHaveBeenCalledTimes(1);
-        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: 16}, {disableDefaultResponse: true});
+        expect(endpoint.command).toHaveBeenCalledWith("manuSpecificTuyaDimmer", "setData", {data: [1,1], dp: 259, fn: 0, status: 0, transid: expect.any(Number)}, {disableDefaultResponse: true});
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/MKS-CM-W5');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state_l3: 'ON'});
@@ -999,4 +1126,58 @@ describe('Entity publish', () => {
         expect(endpoint.command).toHaveBeenCalledTimes(0);
         expect(logger.error).toHaveBeenCalledWith("No converter available for 'get' 'brightness_move' (20)");
     });
+
+    it('Should restore brightness when its turned off without transition and is turned on with', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/1097
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.command.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(1);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "brightness": 123}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(2);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 0}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "OFF"}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(3);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({"state": "ON", "transition": 1.0}));
+        await flushPromises();
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 123, transtime: 10}, {}]);
+    });
+
+    it('Shouldnt use moveToLevelWithOnOff on turn on when no transition has been used as some devices do not turn on in that case', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3332
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'brightness': 150}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'OFF'}));
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({'state': 'ON'}));
+        await flushPromises();
+
+        expect(endpoint.command).toHaveBeenCalledTimes(4);
+        expect(endpoint.command.mock.calls[0]).toEqual(["genOnOff", "on", {}, {}]);
+        expect(endpoint.command.mock.calls[1]).toEqual(["genLevelCtrl", "moveToLevelWithOnOff", {level: 150, transtime: 0}, {}]);
+        expect(endpoint.command.mock.calls[2]).toEqual(["genOnOff", "off", {}, {}]);
+        expect(endpoint.command.mock.calls[3]).toEqual(["genOnOff", "on", {}, {}]);
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish.mock.calls[0]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON"}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[1]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[2]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"OFF","brightness":0}', { qos: 0, retain: false }, expect.any(Function)]);
+        expect(MQTT.publish.mock.calls[3]).toEqual([ 'zigbee2mqtt/bulb_color', '{"state":"ON","brightness":150}', { qos: 0, retain: false }, expect.any(Function)]);
+    });
 });

test/receive.test.js

@@ -8,7 +8,7 @@ const flushPromises = () => new Promise(setImmediate);
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-describe('Device receive', () => {
+describe('Receive', () => {
     let controller;
 
     beforeEach(async () => {
@@ -33,23 +33,6 @@ describe('Device receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-
-    it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', async () => {
-        settings.set(['homeassistant'], true);
-        const device = zigbeeHerdsman.devices.WXKG11LM;
-        const data = {onOff: 1}
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
-        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
-    });
-
     it('Should handle a zigbee message which uses ep (left)', async () => {
         const device = zigbeeHerdsman.devices.WXKG02LM;
         const data = {onOff: 1}
@@ -159,6 +142,20 @@ describe('Device receive', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
     });
 
+    it('Shouldnt republish old state', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3572
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        jest.runAllTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+    });
+
     it('Should handle a zigbee message with 1 precision', async () => {
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
@@ -369,23 +366,27 @@ describe('Device receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from unsupported devices', async () => {
+    it('Should not handle messages from unsupported devices and link to docs', async () => {
         const device = zigbeeHerdsman.devices.unsupported;
         const data = {onOff: 1};
+        logger.warn.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+    it('Should not handle messages from still interviewing devices with unknown definition', async () => {
         const device = zigbeeHerdsman.devices.interviewing;
         const data = {onOff: 1};
+        logger.debug.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, definition is undefined and still interviewing`);
     });
 
     it('Should handle a command', async () => {

test/report.test.js

@@ -19,10 +19,10 @@ const debounce = require('debounce');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
 
-describe('Device report', () => {
+describe('Report', () => {
     let controller;
 
-    function expectOnOffBrightnessColorReport(endpoint) {
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
@@ -31,7 +31,11 @@ describe('Device report', () => {
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
         expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
-        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
     }
 
     mockClear = (device) => {
@@ -57,7 +61,7 @@ describe('Device report', () => {
     it('Should configure reporting on startup', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, true);
     });
 
     it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
@@ -70,7 +74,7 @@ describe('Device report', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
         expect(device.save).toHaveBeenCalledTimes(1);
     });
 
@@ -104,6 +108,20 @@ describe('Device report', () => {
         expect(device.meta.reporting).toBeUndefined();
     });
 
+    it('Should not configure reporting when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        device.interviewing = true;
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
     it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
@@ -130,7 +148,7 @@ describe('Device report', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
-        expectOnOffBrightnessColorReport(endpoint);
+        expectOnOffBrightnessColorReport(endpoint, false);
     });
 
     it('Should not configure reporting on device leave', async () => {
@@ -211,4 +229,31 @@ describe('Device report', () => {
         expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
     });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
 });

test/settings.test.js

@@ -125,6 +125,12 @@ describe('Settings', () => {
 
         settings._write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'user'], 'test123');
+        settings.set(['advanced', 'network_key'], [1,2,3, 4]);
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
     });
 
     it('Should read devices form a separate file', () => {
@@ -353,6 +359,14 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Should throw error when changing entity options of non-existing device', () => {
+        write(configurationFile, {});
+
+        expect(() => {
+            settings.changeEntityOptions('not_existing_123', {});
+        }).toThrow(new Error("Device or group 'not_existing_123' does not exist"));
+    });
+
     it('Should not add duplicate groups', () => {
         write(configurationFile, {});
 
@@ -434,6 +448,31 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Shouldnt crash when removing device from group when group has no devices', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
+                '1': {
+                    friendly_name: 'test123',
+                }
+            }
+        });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
     it('Should throw when adding device to non-existing group', () => {
         write(configurationFile, {
             devices: {
@@ -464,6 +503,19 @@ describe('Settings', () => {
         }).toThrow(new Error("Device '0x123' already exists"));
     });
 
+    it('Should not allow any string values for network_key', () => {
+        write(configurationFile, {
+            advanced: {network_key: 'NOT_GENERATE'},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+    });
+
+
     it('Should not allow retention configuration without MQTT v5', () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
@@ -476,6 +528,19 @@ describe('Settings', () => {
         }).toThrowError('MQTT retention requires protocol version 5');
     });
 
+    it('Should not allow non-existing entities in availability_blocklist', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
+            advanced: {availability_blocklist: ['0x0017880104e45519', 'non_existing']},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blocklist'`);
+    });
+
     it('Should ban devices', () => {
         write(configurationFile, {});
         settings.banDevice('0x123');
@@ -484,6 +549,14 @@ describe('Settings', () => {
         expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
     });
 
+    it('Should add devices to blocklist', () => {
+        write(configurationFile, {});
+        settings.blockDevice('0x123');
+        expect(settings.get().blocklist).toStrictEqual(['0x123']);
+        settings.blockDevice('0x1234');
+        expect(settings.get().blocklist).toStrictEqual(['0x123', '0x1234']);
+    });
+
     it('Should throw error when yaml file is invalid', () => {
         fs.writeFileSync(configurationFile, `
              good: 9
@@ -536,6 +609,18 @@ describe('Settings', () => {
         }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
     });
 
+    it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+    });
+
     it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
         write(configurationFile, {
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
@@ -545,7 +630,7 @@ describe('Settings', () => {
 
         expect(() => {
             settings.validate();
-        }).toThrowError(`Following friendly_name are not allowed: '${utils.getPostfixes()}'`);
+        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {

test/stub/data.js

@@ -5,6 +5,7 @@ const fs = require('fs');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
+const stateFile = path.join(mockDir, 'state.json');
 
 function writeDefaultConfiguration() {
     const config = {
@@ -151,6 +152,9 @@ function writeDefaultConfiguration() {
             '0x0017880104e43559': {
                 friendly_name: 'U202DST600ZB'
             },
+            '0x0017880104e44559': {
+                friendly_name: '3157100_thermostat',
+            }
         },
         groups: {
             '1': {
@@ -171,14 +175,25 @@ function writeDefaultConfiguration() {
                 retain: false,
                 devices: ['bulb_2']
             },
-        }
+        },
+        external_converters: [],
     };
 
     yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
 }
 
 function writeEmptyState() {
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
+    fs.writeFileSync(stateFile, JSON.stringify({}));
+}
+
+function removeState() {
+    if (stateExists()) {
+        fs.unlinkSync(stateFile);
+    }
+}
+
+function stateExists() {
+    return fs.existsSync(stateFile);
 }
 
 function writeDefaultState() {
@@ -197,10 +212,6 @@ function writeDefaultState() {
     fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
 }
 
-function removeState() {
-    fs.unlinkSync(path.join(mockDir, 'state.json'))
-}
-
 jest.mock('../../lib/util/data', () => ({
     joinPath: (file) => require('path').join(mockDir, file),
     getPath: () => mockDir,
@@ -211,8 +222,10 @@ writeDefaultState();
 
 module.exports = {
     mockDir,
+    read: () => yaml.read(path.join(mockDir, 'configuration.yaml')),
     writeDefaultConfiguration,
     writeDefaultState,
     removeState,
     writeEmptyState,
+    stateExists,
 };

test/stub/zigbeeHerdsman.js

@@ -8,6 +8,7 @@ class Group {
         this.meta = {};
         this.members = members;
         this.removeFromDatabase = jest.fn();
+        this.removeFromNetwork = jest.fn();
         this.hasMember = (endpoint) => this.members.includes(endpoint);
     }
 }
@@ -47,9 +48,10 @@ class Endpoint {
             return this.outputClusters.includes(clusters[cluster]);
         }
 
-        this.addToGroup = (group) => {
+        this.addToGroup = jest.fn();
+        this.addToGroup.mockImplementation((group) => {
             if (!group.members.includes(this)) group.members.push(this);
-        }
+        })
 
         this.getDevice = () => {
             return Object.values(devices).find(d => d.ieeeAddr === deviceIeeeAddress);
@@ -100,13 +102,13 @@ class Device {
 
 const returnDevices = [];
 
-const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3')], true, "Mains (single phase)", "LLC020");
+const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
-    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
@@ -139,12 +141,13 @@ const devices = {
     'LIVOLO': new Device('Router', '0x0017880104e45560', 6541,4152, [new Endpoint(6, [0, 6], [])], true, "Mains (single phase)", 'TI0001          '),
     'tradfri_remote': new Device('EndDevice', '0x90fd9ffffe4b64ae', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64ae')], true, "Battery", "TRADFRI remote control"),
     'roller_shutter': new Device('EndDevice', '0x90fd9ffffe4b64af', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64af')], true, "Battery", "SCM-R_00.00.03.15TC"),
-    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax')], true, "Mains (single phase)", "lumi.light.aqcn02"),
+    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "lumi.light.aqcn02"),
     'SP600_OLD': new Device('Router', '0x90fd9ffffe4b64aa', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20160120'),
     'SP600_NEW': new Device('Router', '0x90fd9ffffe4b64ab', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20170220'),
     'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
     'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
 }
 
 const groups = {