1.14.4

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE.md

@@ -3,7 +3,7 @@ Before submitting an issue make sure you've searched for a similar issue and rea
 
 Rules (don't ignore these, your issue will be closed without further notice):
 - English only
-- Make sure you are on the latest Zigbee2mqtt version
+- Make sure you are on the latest Zigbee2MQTT version
 - Provide a clear description of the problem
 - DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
 - Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
@@ -11,7 +11,7 @@ Rules (don't ignore these, your issue will be closed without further notice):
 Did you read the FAQ?
 - https://www.zigbee2mqtt.io/information/FAQ.html
 
-Zigbee2mqtt fails to start?
+Zigbee2MQTT fails to start?
 - https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
 
 Having issues when using a CC2531?
@@ -41,6 +41,6 @@ Bug report?
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-Zigbee2mqtt version:
+Zigbee2MQTT version:
 Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
 Adapter firmware version:

.github/branch-switcher.yml

@@ -1 +1,3 @@
-preferredBranch: dev
+preferredBranch: dev
+exclude:
+  label: branch-switcher-ignore

.github/stale.yml

@@ -1,13 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-# Label to use when marking an issue as stale
-staleLabel: stale
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+        registry-url: https://registry.npmjs.org/
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint
+    - name: Docker login
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
+    - name: Docker setup
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: |
+        sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
+        sudo chmod a+x /usr/local/bin/buildx
+        docker run --rm --privileged hypriot/qemu-register:v2.7.0
+        buildx create --use
+        buildx ls
+    - name: Docker build dev
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest-dev \
+          .
+    - name: Docker build release
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: |
+        TAG="$(git describe --tags)"
+        buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
+          .
+    - name: Publish to npm
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
+    - name: Trigger danielwelch/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 1}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -0,0 +1,18 @@
+name: "Close stale issues/pull requests"
+on:
+  schedule:
+  - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        days-before-stale: 30
+        days-before-close: 7
+        operations-per-run: 500

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee-herdsman" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v2
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}

.npmignore

@@ -7,7 +7,6 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
-azure-pipelines.yaml
 
 #linters
 .jscsrc

README.md

@@ -5,8 +5,8 @@
     <br>
     <br>
     <div style="display: flex;">
-        <a href="https://dev.azure.com/koenkk/zigbee/_build?definitionId=5">
-            <img src="https://dev.azure.com/koenkk/zigbee/_apis/build/status/Koenkk.zigbee2mqtt?branchName=dev">
+        <a href="https://github.com/Koenkk/zigbee2mqtt/actions?query=workflow%3ACI">
+            <img src="https://github.com/koenkk/zigbee2mqtt/workflows/CI/badge.svg">
         </a>
         <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
             <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
@@ -33,7 +33,7 @@
             <img src="https://img.shields.io/npm/v/zigbee2mqtt">
         </a>
     </div>
-    <h1>Zigbee2mqtt  🌉 🐝</h1>
+    <h1>Zigbee2MQTT  🌉 🐝</h1>
     <p>
         Allows you to use your Zigbee devices <b>without</b> the vendors bridge or gateway.
     </p>
@@ -48,12 +48,12 @@ The [documentation](https://www.zigbee2mqtt.io/) provides you all the informatio
 If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/information/zigbee_network.html) to help you out.
 
 ## [Integrations](https://www.zigbee2mqtt.io/#integration)
-Zigbee2mqtt integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
+Zigbee2MQTT integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Hassio](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
@@ -67,7 +67,7 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ![Architecture](images/architecture.png)
 
 ### Internal Architecture
-Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.

azure-pipelines.yaml

@@ -1,88 +0,0 @@
-trigger:
-  tags:
-    include:
-    - '*'
-  branches:
-    include:
-    - '*'
-
-pool:
-  vmImage: 'ubuntu-latest'
-
-steps:
-
-# Prepare
-- task: NodeTool@0
-  inputs:
-    versionSpec: '12.x'
-  displayName: 'Install Node.js'
-- bash: npm ci
-  displayName: 'Install dependencies'
-
-# Verify
-- bash: npm run test-with-coverage
-  displayName: 'Test'
-- bash: npm run eslint
-  displayName: 'Lint'
-
-# NPM
-- task: Npm@1
-  inputs:
-    command: publish
-    publishEndpoint: npm
-  displayName: 'Publish to npm'
-  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
-
-# Docker
-- task: DockerInstaller@0
-  displayName: Docker install
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-  inputs:
-    dockerVersion: 19.03.5
-    releaseType: stable
-- task: Docker@2
-  displayName: Docker registry login
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-  inputs:
-    command: login
-    containerRegistry: docker_registry
-- bash: |
-    sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
-    sudo chmod a+x /usr/local/bin/buildx
-    docker run --rm --privileged hypriot/qemu-register:v2.7.0
-    buildx create --use
-    buildx ls
-  displayName: 'Docker setup'
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-- bash: |
-    buildx build \
-      --build-arg COMMIT=$(git rev-parse --short HEAD) \
-      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
-      -f docker/Dockerfile \
-      --push \
-      -t koenkk/zigbee2mqtt:latest-dev \
-      .
-  displayName: 'Docker build dev'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))
-- bash: |
-    TAG="$(git describe --tags)"
-    buildx build \
-      --build-arg COMMIT=$(git rev-parse --short HEAD) \
-      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
-      -f docker/Dockerfile \
-      --push \
-      -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
-      .
-  displayName: 'Docker build release'
-  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
-
-# Trigger Home Assistant Zigbee2mqtt addon build
-- bash: |
-    curl -s \
-        --request POST \
-        --header "Content-Type: application/json" \
-        --data '{"definition": {"id": 1}}' \
-        -u "$(HASSIO_TRIGGER_SECRET)" \
-        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
-  displayName: 'Trigger Hass.io build'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -3,12 +3,12 @@ FROM node:12-alpine as base
 WORKDIR /app
 RUN apk add --no-cache tzdata eudev
 
-COPY package.json .
+COPY package.json ./
 
 # Dependencies
 FROM base as dependencies
 
-COPY npm-shrinkwrap.json .
+COPY npm-shrinkwrap.json ./
 
 RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
     npm ci --production && \
@@ -19,12 +19,15 @@ FROM base as release
 
 COPY --from=dependencies /app/node_modules ./node_modules
 COPY lib ./lib
-COPY index.js docker/run.sh data/configuration.yaml ./
+COPY index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
-RUN chmod +x /app/run.sh
 RUN mkdir /app/data
 
 ARG COMMIT
 RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
 
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["npm", "start"]

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,4 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-exec npm start
+exec "$@"

index.js

@@ -3,7 +3,22 @@ const engines = require('./package.json').engines;
 
 const version = engines.node;
 if (!semver.satisfies(process.version, version)) {
-    console.log(`\t\tZigbee2mqtt requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+}
+
+// Validate settings
+const settings = require('./lib/util/settings');
+const errors = settings.validate();
+if (errors.length > 0) {
+    console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+    console.log('            READ THIS CAREFULLY\n');
+    console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+    for (const error of errors) {
+        console.log(`- ${error}`);
+    }
+    console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`);
+    console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+    process.exit(1);
 }
 
 const Controller = require('./lib/controller');

lib/controller.js

@@ -6,9 +6,11 @@ const logger = require('./util/logger');
 const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const stringify = require('json-stable-stringify');
 const fs = require('fs');
 const data = require('./util/data');
 const path = require('path');
+const assert = require('assert');
 
 // Extensions
 const ExtensionPublish = require('./extension/publish');
@@ -26,6 +28,14 @@ const ExtensionBind = require('./extension/bind');
 const ExtensionReport = require('./extension/report');
 const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
+
+const AllExtensions = [
+    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
+    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
+    ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters,
+];
 
 class Controller {
     constructor() {
@@ -35,6 +45,7 @@ class Controller {
         this.state = new State(this.eventBus);
 
         this.publishEntityState = this.publishEntityState.bind(this);
+        this.enableDisableExtension = this.enableDisableExtension.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
@@ -49,20 +60,17 @@ class Controller {
             new ExtensionBind(...args),
             new ExtensionOnEvent(...args),
             new ExtensionOTAUpdate(...args),
+            new ExtensionReport(...args),
         ];
 
         if (settings.get().experimental.new_api) {
-            this.extensions.push(new ExtensionBridge(...args));
+            this.extensions.push(new ExtensionBridge(...args, this.enableDisableExtension));
         }
 
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
 
-        if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionReport(...args));
-        }
-
         if (settings.get().homeassistant) {
             this.extensions.push(new ExtensionHomeAssistant(...args));
         }
@@ -75,6 +83,9 @@ class Controller {
         if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
+        }
 
         const extensionPath = data.joinPath('extension');
         if (fs.existsSync(extensionPath)) {
@@ -87,22 +98,10 @@ class Controller {
     }
 
     async start() {
-        const settingsErrors = settings.validate();
-        if (settingsErrors) {
-            logger.error(`Refusing to start, configuration.yaml is not valid, found the following errors:`);
-            for (const error of settingsErrors) {
-                logger.error(`\t - ${error}`);
-            }
-            logger.error(
-                `If you don't know how to solve this, read https://www.zigbee2mqtt.io/configuration/configuration.html`,
-            );
-            process.exit(1);
-        }
-
         this.state.start();
 
         const info = await utils.getZigbee2mqttVersion();
-        logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
+        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
         try {
@@ -122,14 +121,10 @@ class Controller {
         logger.info(`Currently ${devices.length} devices are joined:`);
         for (const device of devices) {
             const entity = this.zigbee.resolveEntity(device);
-            logger.info(
-                (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
-                ` (${entity.device.ieeeAddr}): ` +
-                (entity.definition ?
-                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
-                    'Not supported ') +
-                `(${entity.device.type})`,
-            );
+            const model = entity.definition ?
+                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
+                'Not supported';
+            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
         }
 
         // Enable zigbee join.
@@ -139,14 +134,18 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        await this.zigbee.permitJoin(settings.get().permit_join);
+        try {
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
 
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
         await this.mqtt.connect();
 
         // Send all cached states.
-        if (settings.get().advanced.cache_state) {
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
             for (const device of this.zigbee.getClients()) {
                 if (this.state.exists(device.ieeeAddr)) {
                     this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
@@ -165,6 +164,23 @@ class Controller {
         await this.callExtensionMethod('onMQTTConnected', []);
     }
 
+    async enableDisableExtension(enable, name) {
+        if (!enable) {
+            const extension = this.extensions.find((e) => e.constructor.name === name);
+            if (extension) {
+                await this.callExtensionMethod('stop', [], [extension]);
+                this.extensions.splice(this.extensions.indexOf(extension), 1);
+            }
+        } else {
+            const Extension = AllExtensions.find((e) => e.name === name);
+            assert(Extension, `Extension '${name}' does not exist`);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
+            this.extensions.push(extension);
+            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            this.callExtensionMethod('onMQTTConnected', [], [extension]);
+        }
+    }
+
     async stop() {
         // Call extensions
         await this.callExtensionMethod('stop', []);
@@ -189,9 +205,9 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
+        if (data.device && !settings.getDevice(data.device.ieeeAddr) && data.device.type !== 'Coordinator') {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
 
         const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
@@ -199,7 +215,7 @@ class Controller {
         if (type === 'message') {
             logger.debug(
                 `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
-                `, data '${JSON.stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
                 (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
             );
         } else if (type === 'deviceJoined') {
@@ -266,13 +282,12 @@ class Controller {
             messagePayload = newState;
         }
 
-        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
-            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
+            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
+            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
         };
 
-        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention};
         }
@@ -298,24 +313,18 @@ class Controller {
         }
 
         // filter mqtt message attributes
-        if (deviceOptions.filtered_attributes) {
-            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
-        }
         if (resolvedEntity.settings.filtered_attributes) {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
         if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'attribute_and_json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
+            }
+
+            if (output === 'attribute_and_json' || output === 'attribute') {
                 await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-            } else if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
-            } else {
-                /* istanbul ignore else */
-                if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-                }
             }
         }
 
@@ -325,7 +334,7 @@ class Controller {
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {
         for (const [key, value] of Object.entries(payload)) {
             let subPayload = value;
-            let message;
+            let message = null;
 
             // Special cases
             if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
@@ -333,20 +342,24 @@ class Controller {
             }
 
             // Check Array first, since it is also an Object
-            if (Array.isArray(subPayload)) {
+            if (subPayload === null || subPayload === undefined) {
+                message = '';
+            } else if (Array.isArray(subPayload)) {
                 message = subPayload.map((x) => `${x}`).join(',');
             } else if (typeof subPayload === 'object') {
-                return this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
+                this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
             } else {
-                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
             }
 
-            await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            if (message !== null) {
+                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            }
         }
     }
 
-    async callExtensionMethod(method, parameters) {
-        for (const extension of this.extensions) {
+    async callExtensionMethod(method, parameters, extensions=null) {
+        for (const extension of extensions || this.extensions) {
             if (extension[method]) {
                 try {
                     await extension[method](...parameters);

lib/eventBus.js

@@ -7,18 +7,36 @@ const allowedEvents = [
     'groupRenamed', // Group has been renamed
     'publishEntityState', // Entity state will be published
     'stateChange', // Entity changes its state
+    'groupMembersChanged', // Members of a group has been changed
+    'reportingDisabled', // Reporting is disabled for a device
 ];
 
 class EventBus extends events.EventEmitter {
+    constructor() {
+        super();
+        this.callbackByExtension = {};
+    }
+
     emit(event, data) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
         super.emit(event, data);
     }
 
-    on(event, callback) {
+    on(event, callback, extension=null) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        if (extension) {
+            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
+            this.callbackByExtension[extension].push({event, callback});
+        }
+
         super.on(event, callback);
     }
+
+    removeListenersExtension(extension) {
+        for (const entry of this.callbackByExtension[extension] || []) {
+            super.removeListener(entry.event, entry.callback);
+        }
+    }
 }
 
 module.exports = EventBus;

lib/extension/availability.js

@@ -24,22 +24,27 @@ class Availability extends Extension {
         this.timers = {};
         this.state = {};
 
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
+
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
     }
 
-    inWhitelistOrNotInBlacklist(device) {
+    inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
         const name = deviceSettings.friendlyName;
 
-        // Whitelist is not empty and device is in it, enable availability
-        if (this.whitelist.length > 0) {
-            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
         }
 
-        // Device is on blacklist, disable availability
-        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
             return false;
         }
 
@@ -52,7 +57,7 @@ class Availability extends Extension {
         }
 
         // Device is a mains powered router
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
+        return device.type === 'Router' && device.powerSource !== 'Battery';
     }
 
     onMQTTConnected() {
@@ -61,7 +66,7 @@ class Availability extends Extension {
             const ieeeAddr = device.ieeeAddr;
             this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.inWhitelistOrNotInBlacklist(device)) {
+            if (this.inPasslistOrNotInBlocklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
@@ -119,6 +124,7 @@ class Availability extends Extension {
     }
 
     async stop() {
+        super.stop();
         for (const timer of Object.values(this.timers)) {
             clearTimeout(timer);
         }
@@ -166,7 +172,7 @@ class Availability extends Extension {
             return;
         }
 
-        if (this.inWhitelistOrNotInBlacklist(device)) {
+        if (this.inPasslistOrNotInBlocklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {

lib/extension/bind.js

@@ -1,10 +1,12 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify');
 
-const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
 // See zigbee-herdsman-converters devices.js
 const defaultBindGroup = {type: 'group_number', ID: 901};
@@ -21,99 +23,140 @@ class Bind extends Extension {
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
+        }
     }
 
     parseMQTTMessage(topic, message) {
         let type = null;
         let sourceKey = null;
         let targetKey = null;
+        let clusters = null;
 
         if (this.legacyApi && topic.match(legacyTopicRegex)) {
             topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
             type = topic.split('/')[0];
             sourceKey = topic.replace(`${type}/`, '');
             targetKey = message;
+        } else if (settings.get().experimental.new_api && topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
+            clusters = message.clusters;
         }
 
-        return {type, sourceKey, targetKey};
+        return {type, sourceKey, targetKey, clusters};
     }
 
     async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
 
-        // Find source; can only be a device and target
+        let error = null;
         const source = this.zigbee.resolveEntity(sourceKey);
-        assert(source != null && source.type === 'device', 'Source undefined or not a device');
         const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
-        assert(target != null, 'Target is unknown');
+        const responseData = {from: sourceKey, to: targetKey};
 
-        const sourceName = source.settings.friendlyName;
-        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
-        let attemptedToBindSomething = false;
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
 
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters.
-        for (const cluster of clusters) {
-            const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusterCandidates) {
+                if (clusters && !clusters.includes(cluster)) continue;
 
-            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                attemptedToBindSomething = true;
-                try {
-                    let bindTarget = null;
-                    if (target.type === 'group') bindTarget = target.group;
-                    else if (target.type === 'group_number') bindTarget = target.ID;
-                    else bindTarget = target.endpoint;
+                const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
 
-                    if (type === 'bind') {
-                        await source.endpoint.bind(cluster, bindTarget);
-                    } else {
-                        await source.endpoint.unbind(cluster, bindTarget);
-                    }
+                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
 
-                    logger.info(
-                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceName}' to '${targetName}'`,
-                    );
+                    try {
+                        let bindTarget = null;
+                        if (target.type === 'group') bindTarget = target.group;
+                        else if (target.type === 'group_number') bindTarget = target.ID;
+                        else bindTarget = target.endpoint;
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
                         );
-                    }
-                } catch (error) {
-                    logger.error(
-                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                        `'${targetName}' (${error})`,
-                    );
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}_failed`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
                         );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
                     }
                 }
             }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
         }
 
-        if (!attemptedToBindSomething) {
-            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
+        }
 
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
-                );
-            }
+        if (error) {
+            logger.error(error);
         }
     }
 }

lib/extension/bridge.js

@@ -3,20 +3,33 @@ const utils = require('../util/utils');
 const Extension = require('./extension');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../util/settings');
+const Transport = require('winston-transport');
+const stringify = require('json-stable-stringify');
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
 class Bridge extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-
+        this.enableDisableExtension = enableDisableExtension;
+        this.lastJoinedDeviceIeeeAddr = null;
+        this.setupMQTTLogging();
 
         this.requestLookup = {
-            'permitjoin': this.permitJoin.bind(this),
-            // 'device/remove': this.deviceRemove.bind(this),
-            // 'device/forceremove': this.deviceForceRemove.bind(this),
-            // 'device/ban': this.deviceBan.bind(this),
-            // 'group/remove': this.groupRemove.bind(this),
+            'device/options': this.deviceOptions.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/rename': this.deviceRename.bind(this),
+            'group/add': this.groupAdd.bind(this),
+            'group/options': this.groupOptions.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+            'group/rename': this.groupRename.bind(this),
+            'permit_join': this.permitJoin.bind(this),
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/homeassistant': this.configHomeAssistant.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
+            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'health_check': this.healthCheck.bind(this),
         };
     }
 
@@ -24,34 +37,56 @@ class Bridge extends Extension {
         this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
         this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
     }
 
+    setupMQTTLogging() {
+        const mqtt = this.mqtt;
+        class EventTransport extends Transport {
+            log(info, callback) {
+                const payload = stringify({message: info.message, level: info.level});
+                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
+                callback();
+            }
+        }
+
+        logger.add(new EventTransport());
+    }
+
     async onMQTTMessage(topic, message) {
         const match = topic.match(requestRegex);
         if (match && this.requestLookup[match[1].toLowerCase()]) {
-            message = JSON.parse(message);
+            message = utils.parseJSON(message, message);
+
             try {
                 const response = await this.requestLookup[match[1].toLowerCase()](message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             } catch (error) {
                 logger.error(`Request '${topic}' failed with error: '${error.message}'`);
                 const response = utils.getResponse(message, {}, error.message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             }
         }
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
-        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
+        }
+
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
             let payload;
             const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
-            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
-            else if (type === 'deviceInterview') {
-                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
+            if (type === 'deviceJoined') {
+                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
+            } else if (type === 'deviceInterview') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
+                };
                 if (data.status === 'successful') {
                     payload.supported = !!resolvedEntity.definition;
                     payload.definition = resolvedEntity.definition ? {
@@ -61,9 +96,17 @@ class Bridge extends Extension {
                         supports: resolvedEntity.definition.supports,
                     } : null;
                 }
-            } else payload = {ieeeAddress}; // deviceLeave
+            } else if (type === 'deviceAnnounce') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
+                };
+            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
 
-            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+            await this.mqtt.publish(
+                'bridge/event',
+                stringify({type: utils.toSnakeCase(type), data: payload}),
+                {retain: false, qos: 0},
+            );
         }
 
         if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
@@ -75,93 +118,264 @@ class Bridge extends Extension {
      * Requests
      */
 
-    // async deviceRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('remove', 'device', message);
-    // }
+    async deviceOptions(message) {
+        return this.changeEntityOptions('device', message);
+    }
 
-    // async deviceForceRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
-    // }
+    async groupOptions(message) {
+        return this.changeEntityOptions('group', message);
+    }
 
-    // async deviceBan(message) {
-    //     return this.removeForceRemoveOrBanEntity('ban', 'device', message);
-    // }
+    async deviceRemove(message) {
+        return this.removeEntity('device', message);
+    }
 
-    // async groupRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('remove', 'group', message);
-    // }
+    async groupRemove(message) {
+        return this.removeEntity('group', message);
+    }
+
+    async healthCheck(message) {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    async groupAdd(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    async deviceRename(message) {
+        return this.renameEntity('device', message);
+    }
+
+    async groupRename(message) {
+        return this.renameEntity('group', message);
+    }
 
     async permitJoin(message) {
-        const value = typeof message === 'object' ? message.value : message;
+        const value = this.getValue(message);
         await this.zigbee.permitJoin(value);
         await this.publishInfo();
         return utils.getResponse(message, {value: value}, null);
     }
 
+    configLastSeen(message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configHomeAssistant(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        this.enableDisableExtension(value, 'HomeAssistant');
+        settings.set(['homeassistant'], value);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configElapsed(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        return utils.getResponse(message, {value}, null);
+    }
+
+    configLogLevel(message) {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    async touchlinkFactoryReset(message) {
+        logger.info('Start touchlink factory reset');
+        const result = await this.zigbee.touchlinkFactoryReset();
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, {}, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
     /**
      * Utils
      */
 
-    // async removeForceRemoveOrBanEntity(action, entityType, message) {
-    //     const ID = typeof message === 'object' ? message.ID : message.trim();
-    //     const entity = this.zigbee.resolveEntity(ID);
-    //     if (!entity || entity.type !== entityType) {
-    //         throw new Error(`${ID} is not a ${entityType}`);
-    //     }
+    getValue(message) {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
 
-    //     const lookup = {
-    //         ban: ['banned', 'Banning', 'ban'],
-    //         force_remove: ['force_removed', 'Force removing', 'force remove'],
-    //         remove: ['removed', 'Removing', 'remove'],
-    //     };
+            return message.value;
+        } else {
+            return message;
+        }
+    }
 
-    //     try {
-    //         logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
-    //         if (entity.type === 'device') {
-    //             if (action === 'ban') {
-    //                 settings.banDevice(entity.settings.ID);
-    //             }
+    changeEntityOptions(entityType, message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
 
-    //             action === 'force_remove' ?
-    //                 await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
-    //         } else {
-    //             await entity.group.removeFromDatabase();
-    //         }
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        settings.changeEntityOptions(ID, message.options);
+        const cleanup = (o) => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+        const oldOptions = cleanup(entity.settings);
+        const newOptions = cleanup(settings.getEntity(ID));
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
 
-    //         // Fire event
-    //         if (entity.type === 'device') {
-    //             this.eventBus.emit('deviceRemoved', {device: entity.device});
-    //         }
+    renameEntity(entityType, message) {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
 
-    //         // Remove from configuration.yaml
-    //         entity.type === 'device' ?
-    //             settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
 
-    //         // Remove from state
-    //         this.state.remove(entity.settings.ID);
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const entity = this.getEntity(entityType, from);
 
-    //         logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-    //         entity.type === 'device' ? this.publishDevices() : this.publishGroups();
-    //         return utils.getResponse(message, {ID}, null);
-    //     } catch (error) {
-    //         throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
-    //     }
-    // }
+        settings.changeFriendlyName(from, to);
+
+        if (entity.type === 'device') {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device});
+        } else {
+            this.publishGroups();
+            this.eventBus.emit(`groupRenamed`, {group: entity.group});
+        }
+
+        return utils.getResponse(message, {from: entity.settings.friendlyName, to}, null);
+    }
+
+    async removeEntity(entityType, message) {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity.type === 'device') {
+                if (block) {
+                    settings.blockDevice(entity.settings.ID);
+                }
+
+                if (force) {
+                    await entity.device.removeFromDatabase();
+                } else {
+                    await entity.device.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    await entity.group.removeFromDatabase();
+                } else {
+                    await entity.group.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {device: entity.device});
+            }
+
+            // Remove from configuration.yaml
+            if (entity.type === 'device') {
+                settings.removeDevice(entity.settings.ID);
+            } else {
+                settings.removeGroup(entity.settings.ID);
+            }
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+
+            if (entity.type === 'device') {
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type, ID) {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
 
     async publishInfo() {
         const payload = {
             version: this.zigbee2mqttVersion.version,
             commit: this.zigbee2mqttVersion.commitHash,
             coordinator: this.coordinatorVersion,
-            logLevel: logger.getLevel(),
-            permitJoin: await this.zigbee.getPermitJoin(),
+            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
         };
 
-        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/info', stringify(payload), {retain: true, qos: 0});
     }
 
-    async publishDevices(topic, message) {
-        const devices = this.zigbee.getClients().map((device) => {
+    async publishDevices() {
+        const devices = this.zigbee.getDevices().map((device) => {
             const definition = zigbeeHerdsmanConverters.findByDevice(device);
             const resolved = this.zigbee.resolveEntity(device);
             const definitionPayload = definition ? {
@@ -171,40 +385,68 @@ class Bridge extends Extension {
                 supports: definition.supports,
             } : null;
 
+            const endpoints = {};
+            for (const endpoint of device.endpoints) {
+                const data = {
+                    bindings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    let target;
+
+                    if (bind.target.constructor.name === 'Endpoint') {
+                        target = {
+                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
+                        };
+                    } else {
+                        target = {type: 'group', id: bind.target.groupID};
+                    }
+
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
+
             return {
-                ieeeAddress: device.ieeeAddr,
+                ieee_address: device.ieeeAddr,
                 type: device.type,
-                networkAddress: device.networkAddress,
+                network_address: device.networkAddress,
                 supported: !!definition,
-                friendlyName: resolved.name,
+                friendly_name: resolved.name,
                 definition: definitionPayload,
-                powerSource: device.powerSource,
-                softwareBuildID: device.softwareBuildID,
-                dateCode: device.dateCode,
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
                 interviewing: device.interviewing,
-                interviewCompleted: device.interviewCompleted,
+                interview_completed: device.interviewCompleted,
+                endpoints,
             };
         });
 
-        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/devices', stringify(devices), {retain: true, qos: 0});
     }
 
-    async publishGroups(topic, message) {
+    async publishGroups() {
         const groups = this.zigbee.getGroups().map((group) => {
             const resolved = this.zigbee.resolveEntity(group);
             return {
-                ID: group.groupID,
-                friendlyName: resolved.name,
+                id: group.groupID,
+                friendly_name: resolved.name,
                 members: group.members.map((m) => {
                     return {
-                        ieeeAddress: m.deviceIeeeAddress,
+                        ieee_address: m.deviceIeeeAddress,
                         endpoint: m.ID,
                     };
                 }),
             };
         });
 
-        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+        await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0});
     }
 }
 

lib/extension/configure.js

@@ -1,6 +1,8 @@
 const settings = require('../util/settings');
+const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -10,10 +12,27 @@ class Configure extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         this.configuring = new Set();
+        this.onReportingDisabled = this.onReportingDisabled.bind(this);
         this.attempts = {};
 
-        this.legacyApi = settings.get().advanced.legacy_api;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
+    }
+
+    onReportingDisabled(data) {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        const device = data.device;
+
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity.device.meta && resolvedEntity.device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
     }
 
     shouldConfigure(resolvedEntity) {
@@ -36,18 +55,18 @@ class Configure extends Extension {
 
     onMQTTConnected() {
         /* istanbul ignore else */
-        if (this.legacyApi) {
+        if (settings.get().advanced.legacy_api) {
             this.mqtt.subscribe(this.legacyTopic);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            if (topic !== this.legacyTopic) {
-                return;
-            }
-
+        if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);
             if (!resolvedEntity || resolvedEntity.type !== 'device') {
                 logger.error(`Device '${message}' does not exist`);
@@ -60,6 +79,26 @@ class Configure extends Extension {
             }
 
             this.configure(resolvedEntity, true);
+        } else if (topic === this.topic) {
+            message = utils.parseJSON(message, message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const resolvedEntity = this.zigbee.resolveEntity(ID);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                error = `Device '${ID}' does not exist`;
+            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                error = `Device '${resolvedEntity.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(resolvedEntity, true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
         }
     }
 
@@ -87,7 +126,7 @@ class Configure extends Extension {
         }
     }
 
-    async configure(resolvedEntity, force=false) {
+    async configure(resolvedEntity, force=false, thowError=false) {
         const device = resolvedEntity.device;
         if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
             return false;
@@ -106,11 +145,14 @@ class Configure extends Extension {
             device.meta.configured = resolvedEntity.definition.meta.configureKey;
             device.save();
         } catch (error) {
-            logger.error(
-                `Failed to configure '${resolvedEntity.name}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
             this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
         }
 
         this.configuring.delete(device.ieeeAddr);

lib/extension/extension.js

@@ -46,7 +46,9 @@ class Extension {
     /**
      * Is called once the extension has to stop
      */
-    // stop() {}
+    stop() {
+        this.eventBus.removeListenersExtension(this.constructor.name);
+    }
 }
 
 module.exports = Extension;

lib/extension/externalConverters.js

@@ -0,0 +1,18 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+
+class ExternalConverters extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/groups.js

@@ -3,7 +3,10 @@ const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
 const postfixes = utils.getEndpointNames();
+const stringify = require('json-stable-stringify');
 
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
@@ -26,16 +29,37 @@ class Groups extends Extension {
                 this.mqtt.subscribe(`${topic}remove_all`);
             }
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
+        }
     }
 
     async onZigbeeStarted() {
-        await this.syncGroupsWithSettings();
         this.eventBus.on('stateChange', this.onStateChange);
+        await this.syncGroupsWithSettings();
     }
 
     async syncGroupsWithSettings() {
         const settingsGroups = settings.getGroups();
         const zigbeeGroups = this.zigbee.getGroups();
+
+        const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
+            try {
+                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
+                if (action === 'remove') {
+                    await endpoint.removeFromGroup(group);
+                } else {
+                    await endpoint.addToGroup(group);
+                }
+            } catch (error) {
+                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+            }
+        };
+
         for (const settingGroup of settingsGroups) {
             const groupID = settingGroup.ID;
             const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
@@ -48,28 +72,24 @@ class Groups extends Extension {
             // In settings but not in zigbee
             for (const entity of settingsEntity) {
                 if (!zigbeeGroup.hasMember(entity.endpoint)) {
-                    logger.info(`Adding '${entity.name}' to group '${settingGroup.friendlyName}'`);
-                    await entity.endpoint.addToGroup(zigbeeGroup);
+                    addRemoveFromGroup('add', entity.name, settingGroup.friendlyName, entity.endpoint, zigbeeGroup);
                 }
             }
 
             // In zigbee but not in settings
             for (const endpoint of zigbeeGroup.members) {
                 if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
-                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
-                    logger.info(`Removing '${deviceSettings.friendlyName}' from group '${settingGroup.friendlyName}'`);
-                    await endpoint.removeFromGroup(zigbeeGroup);
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, settingGroup.friendlyName, endpoint, zigbeeGroup);
                 }
             }
         }
 
-        // eslint-disable-next-line
         for (const zigbeeGroup of zigbeeGroups) {
             if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
                 for (const endpoint of zigbeeGroup.members) {
-                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
-                    logger.info(`Removing '${deviceSettings.friendlyName}' from group ${zigbeeGroup.groupID}`);
-                    await endpoint.removeFromGroup(zigbeeGroup);
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.groupID, endpoint, zigbeeGroup);
                 }
             }
         }
@@ -143,32 +163,40 @@ class Groups extends Extension {
         let resolvedEntityGroup = null;
         let resolvedEntityDevice = null;
         let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
 
         /* istanbul ignore else */
-        if (this.legacyApi) {
-            const topicMatch = topic.match(legacyTopicRegex);
-            if (topicMatch) {
-                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
-                type = topicMatch[2];
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
 
                 if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
-                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
                         );
                     }
 
                     return {};
                 }
-            } else if (topic.match(legacyTopicRegexRemoveAll)) {
-                type = 'remove_all';
             } else {
-                return {};
+                type = 'remove_all';
             }
 
             resolvedEntityDevice = this.zigbee.resolveEntity(message);
@@ -177,10 +205,12 @@ class Groups extends Extension {
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
                     );
                 }
 
@@ -188,78 +218,125 @@ class Groups extends Extension {
             }
 
             hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
         }
 
-        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi,
+        };
     }
 
     async onMQTTMessage(topic, message) {
-        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
-            this.parseMQTTMessage(topic, message);
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey,
+        } = this.parseMQTTMessage(topic, message);
         if (!type) return;
 
-        const keys = [
-            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
-            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
-        ];
-
-        const definition = resolvedEntityDevice.definition;
-        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
-        const endpointName = endpoints ?
-            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
-
-        if (endpointName) {
-            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
-            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
         }
 
-        if (!hasEndpointName) {
-            keys.push(resolvedEntityDevice.name);
-            keys.push(resolvedEntityDevice.device.ieeeAddr);
-        }
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
 
-        if (type === 'add') {
-            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
-            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                const endpointName = endpoints ?
+                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
 
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_add`, message: payload}),
-                );
-            }
-        } else if (type === 'remove') {
-            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
-            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_remove`, message: payload}),
-                );
-            }
-        } else { // remove_all
-            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
-            await resolvedEntityDevice.endpoint.removeFromAllGroups();
-            for (const settingsGroup of settings.getGroups()) {
-                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: resolvedEntityDevice.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
-                    );
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
                 }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
             }
         }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit('groupMembersChanged', {group: resolvedEntityGroup});
+        }
     }
 }
 

lib/extension/legacy/bridgeLegacy.js

@@ -4,6 +4,7 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const utils = require('../../util/utils');
 const assert = require('assert');
 const Extension = require('../extension');
+const stringify = require('json-stable-stringify');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
@@ -52,6 +53,7 @@ class BridgeLegacy extends Extension {
             'device_options': this.deviceOptions,
             'add_group': this.addGroup,
             'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
             'whitelist': this.whitelist,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
         };
@@ -65,7 +67,7 @@ class BridgeLegacy extends Extension {
             logger.info(`Whitelisted '${entity.friendlyName}'`);
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
             );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
@@ -88,8 +90,8 @@ class BridgeLegacy extends Extension {
 
         const entity = settings.getEntity(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
-        settings.changeDeviceOptions(entity.ID, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
+        settings.changeEntityOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
     }
 
     async permitJoin(topic, message) {
@@ -175,9 +177,9 @@ class BridgeLegacy extends Extension {
         });
 
         if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
+            this.mqtt.publish(`bridge/config/devices`, stringify(devices), {});
         } else {
-            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
+            this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
         }
     }
 
@@ -188,7 +190,7 @@ class BridgeLegacy extends Extension {
             return group;
         });
 
-        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
+        this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
@@ -232,7 +234,7 @@ class BridgeLegacy extends Extension {
 
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+                stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
             );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
@@ -264,7 +266,7 @@ class BridgeLegacy extends Extension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
+        this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
@@ -272,9 +274,15 @@ class BridgeLegacy extends Extension {
         const name = message;
         const entity = this.zigbee.resolveEntity(message);
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
         settings.removeGroup(message);
-        entity.group.removeFromDatabase();
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
+
+        this.mqtt.publish('bridge/log', stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -301,7 +309,7 @@ class BridgeLegacy extends Extension {
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
@@ -316,7 +324,7 @@ class BridgeLegacy extends Extension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -333,7 +341,7 @@ class BridgeLegacy extends Extension {
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -371,11 +379,12 @@ class BridgeLegacy extends Extension {
             version: info.version,
             commit: info.commitHash,
             coordinator,
+            network: await this.zigbee.getNetworkParameters(),
             log_level: logger.getLevel(),
             permit_join: await this.zigbee.getPermitJoin(),
         };
 
-        await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
@@ -386,7 +395,7 @@ class BridgeLegacy extends Extension {
         if (type === 'deviceJoined') {
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+                stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
             );
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
@@ -395,20 +404,20 @@ class BridgeLegacy extends Extension {
                     const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta: log}),
                     );
                 } else {
                     const meta = {friendly_name: resolvedEntity.name, supported: false};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta}),
                     );
                 }
             } else if (data.status === 'failed') {
                 const meta = {friendly_name: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                    stringify({type: `pairing`, message: 'interview_failed', meta}),
                 );
             } else {
                 /* istanbul ignore else */
@@ -416,13 +425,13 @@ class BridgeLegacy extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                        stringify({type: `pairing`, message: 'interview_started', meta}),
                     );
                 }
             }
         } else if (type === 'deviceAnnounce') {
             const meta = {friendly_name: resolvedEntity.name};
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_announced`, message: 'announce', meta}));
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
@@ -430,7 +439,7 @@ class BridgeLegacy extends Extension {
                 const meta = {friendly_name: name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                    stringify({type: `device_removed`, message: 'left_network', meta}),
                 );
             }
         }
@@ -440,7 +449,7 @@ class BridgeLegacy extends Extension {
         logger.info('Starting touchlink factory reset...');
         this.mqtt.publish(
             'bridge/log',
-            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+            stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
         );
         const result = await this.zigbee.touchlinkFactoryReset();
 
@@ -448,13 +457,13 @@ class BridgeLegacy extends Extension {
             logger.info('Successfully factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+                stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
             );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+                stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
             );
         }
     }

lib/extension/legacy/deviceGroupMembership.js

@@ -29,7 +29,18 @@ class DeviceGroupMembership extends Extension {
             `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
         );
 
-        const {grouplist, capacity} = response;
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendlyName : gid;
+        });
+
         const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
         let msgCapacity;
         if (capacity === 254) {

lib/extension/networkMap.js

@@ -2,6 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify');
 
 /**
  * This extension creates a network map
@@ -13,15 +14,18 @@ class NetworkMap extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
         this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
 
         // Bind
         this.raw = this.raw.bind(this);
         this.graphviz = this.graphviz.bind(this);
+        this.plantuml = this.plantuml.bind(this);
 
         // Set supported formats
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
         };
     }
 
@@ -31,6 +35,10 @@ class NetworkMap extends Extension {
             this.mqtt.subscribe(this.legacyTopic);
             this.mqtt.subscribe(this.legacyTopicRoutes);
         }
+
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(this.topic);
+        }
     }
 
     async onMQTTMessage(topic, message) {
@@ -44,10 +52,33 @@ class NetworkMap extends Extension {
                 this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
             }
         }
+
+        if (settings.get().experimental.new_api && topic === this.topic) {
+            try {
+                message = utils.parseJSON(message, message);
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
+        }
     }
 
     raw(topology) {
-        return JSON.stringify(topology);
+        return stringify(topology);
     }
 
     graphviz(topology) {
@@ -83,7 +114,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+                lastSeen = utils.formatDate(date, 'relative');
             }
 
             labels.push(lastSeen);
@@ -126,6 +157,64 @@ class NetworkMap extends Extension {
         return text.replace(/\0/g, '');
     }
 
+    plantuml(topology) {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
         const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');

lib/extension/onEvent.js

@@ -19,6 +19,7 @@ class OnEvent extends Extension {
     }
 
     async stop() {
+        super.stop();
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
             this.callOnEvent(resolvedEntity, 'stop', {});

lib/extension/otaUpdate.js

@@ -1,7 +1,11 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const stringify = require('json-stable-stringify');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
 const Extension = require('./extension');
 const MINUTES_10 = 1000 * 60 * 10;
 
@@ -19,6 +23,23 @@ class OTAUpdate extends Extension {
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
             this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
         }
+
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+            // remove them.
+            this.removeProgressAndRemainingFromState(device);
+        }
+    }
+
+    removeProgressAndRemainingFromState(device) {
+        this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
+        this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
@@ -35,7 +56,8 @@ class OTAUpdate extends Extension {
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
             const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
-            this.publishEntityState(data.device.ieeeAddr, {update_available: available});
+            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+            this.publishEntityState(data.device.ieeeAddr, payload);
 
             if (available) {
                 const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
@@ -46,7 +68,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        stringify({type: `ota_update`, message, meta}),
                     );
                 }
             }
@@ -79,141 +101,176 @@ class OTAUpdate extends Extension {
         }
     }
 
-    async onMQTTMessage(topic, message) {
-        let resolvedEntity = null;
+    getEntityPublishPayload(state, progress=null, remaining=null) {
+        const payload = {};
+
         /* istanbul ignore else */
         if (this.legacyApi) {
-            if (!topic.match(legacyTopicRegex)) {
-                return null;
-            }
+            payload.update_available = state === 'available';
+        }
 
-            resolvedEntity = this.zigbee.resolveEntity(message);
-        } else {
+        /* istanbul ignore else */
+        if (settings.get().experimental.new_api) {
+            payload.update = {state};
+            if (progress !== null) payload.update.progress = progress;
+            if (remaining !== null) payload.update.remaining = Math.round(remaining);
+        }
+
+        return payload;
+    }
+
+    async onMQTTMessage(topic, message) {
+        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
             return null;
         }
 
-        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
-        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
-            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
-            logger.error(message);
+        message = utils.parseJSON(message, message);
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+        const resolvedEntity = this.zigbee.resolveEntity(ID);
+        const type = topic.substring(topic.lastIndexOf('/') + 1);
+        const responseData = {id: ID};
+        let error = null;
+
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            error = `Device '${ID}' does not exist`;
+        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
                 const meta = {status: `not_supported`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
+                    stringify({type: `ota_update`, message: error, meta}),
                 );
             }
+        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else {
+            this.inProgress.add(resolvedEntity.device.ieeeAddr);
 
-            return;
-        }
-
-        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
-            return;
-        }
-        this.inProgress.add(resolvedEntity.device.ieeeAddr);
-
-        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
-        if (type === 'check') {
-            const message = `Checking if update available for '${resolvedEntity.name}'`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
-                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
-                logger.info(message);
+            if (type === 'check') {
+                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
-                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
-            } catch (error) {
-                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
-                logger.error(message);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `check_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                try {
+                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
+                        resolvedEntity.device, logger,
                     );
-                }
-            }
-        } else { // type === 'update'
-            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const onProgress = (progress, remaining) => {
-                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
-                    if (remaining) {
-                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
-                    }
-
-                    logger.info(message);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
-                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: msg, meta}),
+                        );
                     }
-                };
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
-                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
-                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${resolvedEntity.name}'` +
-                    (to ? `, from '${fromS}' to '${toS}'` : ``);
-                logger.info(message);
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+                    const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
 
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
                 }
-            } catch (error) {
-                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
-                logger.error(message);
+            } else { // type === 'update'
+                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress, remaining) => {
+                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        const payload = this.getEntityPublishPayload('updating', progress, remaining);
+                        this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
+                    }
+                } catch (e) {
+                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('available');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
+                    }
                 }
             }
+
+            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
         }
 
-        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        }
     }
 }
 

lib/extension/publish.js

@@ -5,20 +5,25 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify');
 
 const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 
 const groupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_color,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
     zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
     zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
 ];
 
 class EntityPublish extends Extension {
@@ -64,7 +69,7 @@ class EntityPublish extends Extension {
                 const message = {friendly_name: entityKey};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `entity_not_found`, message}),
+                    stringify({type: `entity_not_found`, message}),
                 );
             }
 
@@ -93,22 +98,34 @@ class EntityPublish extends Extension {
             converters = resolvedEntity.definition.toZigbee;
             options = resolvedEntity.settings;
         } else {
-            converters = groupConverters;
             target = resolvedEntity.group;
             options = resolvedEntity.settings;
-            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
+            definition = resolvedEntity.group.members
+                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
+            converters = new Set(groupConverters);
+            for (const d of definition) {
+                d.toZigbee.forEach(converters.add, converters);
+            }
+            converters = [...converters];
         }
 
         // Convert the MQTT message to a Zigbee message.
         let json = {};
         if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            json[topic.attribute] = message;
+            try {
+                json[topic.attribute] = JSON.parse(message);
+            } catch (e) {
+                json[topic.attribute] = message;
+            }
         } else {
             try {
                 json = JSON.parse(message);
             } catch (e) {
-                // Cannot be parsed to JSON, assume state message.
-                json = {state: message};
+                if (stateValues.includes(message.toLowerCase())) {
+                    json = {state: message};
+                } else {
+                    logger.error(`Invalid JSON '${message}', skipping...`);
+                }
             }
         }
 
@@ -241,7 +258,7 @@ class EntityPublish extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                        stringify({type: `zigbee_publish_error`, message, meta}),
                     );
                 }
             }

lib/extension/receive.js

@@ -3,6 +3,7 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify');
 
 class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
@@ -78,33 +79,14 @@ class Receive extends Extension {
             return false;
         }
 
-        /**
-         * Don't handle re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The source address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the source address of the end device.
-         * As the same message is also received directly from the end device, it makes no sense
-         * to handle these messages.
-         */
-        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
-        if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
-            logger.debug('Skipping re-transmitted Xiaomi message');
-            return false;
-        }
-
-        if (!data.device.modelID && data.device.interviewing) {
-            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
-            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
             return false;
         }
 
@@ -112,6 +94,24 @@ class Receive extends Extension {
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
+            resolvedEntity = this.zigbee.resolveEntity(data.device);
+        }
+
         if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
@@ -125,14 +125,11 @@ class Receive extends Extension {
         if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
-                `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
+                `and type '${data.type}' and data '${stringify(data.data)}'`,
             );
             return;
         }
 
-        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
-        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
-
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
         // - If a payload is returned publish it to the MQTT broker
@@ -154,18 +151,22 @@ class Receive extends Extension {
             }
 
             // Check if we have to debounce
-            if (debounce) {
-                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
+            if (resolvedEntity.settings.debounce) {
+                this.publishDebounce(
+                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
+                    resolvedEntity.settings.debounce_ignore,
+                );
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
             }
         };
 
-        const meta = {device: data.device};
+        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
         let payload = {};
         converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
+            const converted = converter.convert(
+                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
+            );
             if (converted) {
                 payload = {...payload, ...converted};
             }

lib/extension/report.js

@@ -1,5 +1,6 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
+const settings = require('../util/settings');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
 const utils = require('../util/utils');
 const Extension = require('./extension');
@@ -106,6 +107,7 @@ class Report extends Extension {
         this.queue = new Set();
         this.failed = new Set();
         this.pollDebouncers = {};
+        this.enabled = settings.get().advanced.report;
     }
 
     shouldIgnoreClusterForDevice(cluster, definition) {
@@ -124,34 +126,46 @@ class Report extends Extension {
         if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
         this.queue.add(device.ieeeAddr);
 
+        const term1 = this.enabled ? 'Setup' : 'Disable';
+        const term2 = this.enabled ? 'setup' : 'disabled';
+
         try {
             for (const ep of device.endpoints) {
                 for (const [cluster, configuration] of Object.entries(clusters)) {
                     if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
-                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
 
                         const items = [];
                         for (const entry of configuration) {
                             if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
-                                items.push({...entry});
+                                const toAdd = {...entry};
+                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
+                                items.push(toAdd);
                                 delete items[items.length - 1].condition;
                             }
                         }
 
-                        await ep.bind(cluster, this.coordinatorEndpoint);
+                        this.enabled ?
+                            await ep.bind(cluster, this.coordinatorEndpoint) :
+                            await ep.unbind(cluster, this.coordinatorEndpoint);
+
                         await ep.configureReporting(cluster, items);
                         logger.info(
-                            `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
                         );
                     }
                 }
             }
 
-            // eslint-disable-next-line
-            device.meta.reporting = reportKey;
+            if (this.enabled) {
+                device.meta.reporting = reportKey;
+            } else {
+                delete device.meta.reporting;
+                this.eventBus.emit('reportingDisabled', {device});
+            }
         } catch (error) {
             logger.error(
-                `Failed to setup reporting for '${device.ieeeAddr}' - ${error.stack}`,
+                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
             );
 
             this.failed.add(device.ieeeAddr);
@@ -171,12 +185,21 @@ class Report extends Extension {
         // Ikea TRADFRI tend to forget their reporting after powered off.
         // Re-setup reporting.
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
+        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
 
-        if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
-        if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
+        if (resolvedEntity.device.interviewing === true) return false;
+        if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
         // Gledopto devices don't support reporting.
         if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+
+        if (this.enabled && device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) {
+            return false;
+        }
+
+        if (!this.enabled && !device.meta.hasOwnProperty('reporting')) {
+            return false;
+        }
+
         return true;
     }
 
@@ -186,14 +209,14 @@ class Report extends Extension {
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
             if (this.shouldSetupReporting(resolvedEntity, null)) {
-                this.setupReporting(resolvedEntity);
+                await this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, resolvedEntity) {
+    async onZigbeeEvent(type, data, resolvedEntity) {
         if (this.shouldSetupReporting(resolvedEntity, type)) {
-            this.setupReporting(resolvedEntity);
+            await this.setupReporting(resolvedEntity);
         }
 
         if (type === 'message') {

lib/mqtt.js

@@ -99,17 +99,21 @@ class MQTT extends events.EventEmitter {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
         if (!this.isConnected()) {
-            logger.error(`Not connected to MQTT server!`);
-            logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            if (!skipLog) {
+                logger.error(`Not connected to MQTT server!`);
+                logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            }
             return;
         }
 
-        logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        if (!skipLog) {
+            logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        }
 
         return new Promise((resolve) => {
             this.client.publish(topic, payload, options, () => resolve());

lib/state.js

@@ -1,7 +1,9 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
+const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
+const stringify = require('json-stable-stringify');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
@@ -53,9 +55,13 @@ class State {
     }
 
     save() {
-        logger.debug(`Saving state to file ${this.file}`);
-        const json = JSON.stringify(this.state, null, 4);
-        fs.writeFileSync(this.file, json, 'utf8');
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = stringify(this.state, null, 4);
+            fs.writeFileSync(this.file, json, 'utf8');
+        } else {
+            logger.debug(`Not saving state`);
+        }
     }
 
     exists(ID) {
@@ -82,6 +88,24 @@ class State {
         this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
+    removeKey(ID, path) {
+        if (this.exists(ID)) {
+            let state = this.state[ID];
+            for (let i = 0; i < path.length; i++) {
+                const key = path[i];
+                if (i === path.length - 1) {
+                    delete state[key];
+                } else {
+                    if (state[key]) {
+                        state = state[key];
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
     remove(ID) {
         if (this.exists(ID)) {
             delete this.state[ID];

lib/util/logger.js

@@ -46,7 +46,7 @@ const transportsToUse = [
             winston.format.printf(/* istanbul ignore next */(info) => {
                 let {timestamp, level, message} = info;
                 level = level === 'warning' ? 'warn' : level;
-                const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
+                const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
         ),
@@ -82,7 +82,14 @@ if (output.includes('file')) {
 /* istanbul ignore next */
 if (output.includes('syslog')) {
     require('winston-syslog').Syslog;
-    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    const options = {
+        app_name: 'Zigbee2MQTT',
+        format: winston.format.printf(/* istanbul ignore next */(info) => {
+            return `${info.message}`;
+        }),
+        ...settings.get().advanced.log_syslog,
+    };
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
     transportsToUse.push(new winston.transports.Syslog(options));
 }
 

lib/util/settings.js

@@ -8,6 +8,9 @@ const Ajv = require('ajv');
 const ajv = new Ajv({allErrors: true});
 
 const defaults = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
     whitelist: [],
     ban: [],
     permit_join: false,
@@ -55,12 +58,13 @@ const defaults = {
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
-        baudrate: 115200,
-        rtscts: true,
         adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
         availability_blacklist: [],
         availability_whitelist: [],
 
@@ -68,11 +72,13 @@ const defaults = {
          * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
          * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
          *
-         * Therefore zigbee2mqtt BY DEFAULT caches all values and resend it with every message.
+         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
          * advanced.cache_state in configuration.yaml allows to configure this.
          * https://www.zigbee2mqtt.io/configuration/configuration.html
          */
         cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
 
         /**
          * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
@@ -122,6 +128,7 @@ const defaults = {
          */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
+    external_converters: [],
 };
 
 const schema = {
@@ -156,12 +163,16 @@ const schema = {
                 adapter: {type: 'string', enum: ['deconz', 'zstack']},
             },
         },
-        ban: {type: 'array', items: {type: 'string'}},
+        blocklist: {type: 'array', items: {type: 'string'}},
+        passlist: {type: 'array', items: {type: 'string'}},
+        // Deprecated: use block/passlist
         whitelist: {type: 'array', items: {type: 'string'}},
+        ban: {type: 'array', items: {type: 'string'}},
         experimental: {
             type: 'object',
             properties: {
                 transmit_power: {type: 'number'},
+                output: {type: 'string', enum: ['attribute_and_json', 'attribute', 'json']},
             },
         },
         advanced: {
@@ -172,6 +183,8 @@ const schema = {
                 ext_pan_id: {type: 'array', items: {type: 'number'}},
                 channel: {type: 'number', minimum: 11, maximum: 26},
                 cache_state: {type: 'boolean'},
+                cache_state_persistent: {type: 'boolean'},
+                cache_state_send_on_startup: {type: 'boolean'},
                 log_rotation: {type: 'boolean'},
                 log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
                 log_output: {type: 'array', items: {type: 'string'}},
@@ -184,6 +197,9 @@ const schema = {
                 last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
                 elapsed: {type: 'boolean'},
                 availability_timeout: {type: 'number', minimum: 0},
+                availability_blocklist: {type: 'array', items: {type: 'string'}},
+                availability_passlist: {type: 'array', items: {type: 'string'}},
+                // Deprecated, use block/passlist
                 availability_blacklist: {type: 'array', items: {type: 'string'}},
                 availability_whitelist: {type: 'array', items: {type: 'string'}},
                 report: {type: 'boolean'},
@@ -271,6 +287,10 @@ const schema = {
             },
         },
     },
+    external_converters: {
+        type: 'array',
+        items: {type: 'string'},
+    },
     required: ['homeassistant', 'permit_join', 'mqtt'],
 };
 
@@ -313,23 +333,40 @@ function write() {
 }
 
 function validate() {
-    const validate = ajv.compile(schema);
-    const valid = validate(_settings);
-    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
-        _settings.advanced.network_key !== 'GENERATE') {
-        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    try {
+        get();
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            return [
+                `Your YAML file: '${error.file}' is invalid ` +
+                `(use https://jsonformatter.org/yaml-validator to find and fix the issue)`,
+            ];
+        }
+
+        return [error.message];
     }
 
-    const postfixes = utils.getEndpointNames();
+    const validate = ajv.compile(schema);
+    validate(_settings);
+    if (validate.errors) {
+        return validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+    }
+
+    const errors = [];
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
 
     // Verify that all friendly names are unique
     const names = [];
+    const postfixes = utils.getEndpointNames();
     const check = (name) => {
-        if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
-        if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
-        if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
+        if (postfixes.includes(name)) errors.push(`Following friendly_name are not allowed: '${postfixes}'`);
+        if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
         if (name.includes('#') || name.includes('+')) {
-            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+            errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
         }
         names.push(name);
     };
@@ -341,7 +378,7 @@ function validate() {
     if (settingsWithDefaults.mqtt.version !== 5) {
         for (const device of Object.values(settingsWithDefaults.devices)) {
             if (device.retention) {
-                throw new Error('MQTT retention requires protocol version 5');
+                errors.push('MQTT retention requires protocol version 5');
             }
         }
     }
@@ -349,15 +386,17 @@ function validate() {
     const checkAvailabilityList = (list, type) => {
         list.forEach((e) => {
             if (!getEntity(e)) {
-                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+                errors.push(`Non-existing entity '${e}' specified in '${type}'`);
             }
         });
     };
 
     checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
     checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
 
-    return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
+    return errors;
 }
 
 function read() {
@@ -399,9 +438,52 @@ function read() {
     return s;
 }
 
+function applyEnvironmentVariables(settings) {
+    const iterate = (obj, path) => {
+        Object.keys(obj).forEach((key) => {
+            if (key !== 'type') {
+                if (key !== 'properties') {
+                    const type = (obj[key].type || 'object').toString();
+                    const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
+                    const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
+                    if (process.env[envVariableName]) {
+                        const setting = path.reduce((acc, val, index) => {
+                            acc[val] = acc[val] || {};
+                            return acc[val];
+                        }, settings);
+
+                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
+                            setting[key] = JSON.parse(process.env[envVariableName]);
+                        }
+                        if (type.indexOf('number') >= 0) {
+                            setting[key] = process.env[envVariableName] * 1;
+                        }
+                        if (type.indexOf('boolean') >= 0) {
+                            setting[key] = process.env[envVariableName].toLowerCase() === 'true';
+                        }
+                        if (type.indexOf('string') >= 0) {
+                            setting[key] = process.env[envVariableName];
+                        }
+                    }
+                }
+
+                if (typeof obj[key] === 'object') {
+                    const newPath = [...path];
+                    if (key !== 'properties') {
+                        newPath.push(key);
+                    }
+                    iterate(obj[key], newPath);
+                }
+            }
+        });
+    };
+    iterate(schema.properties, []);
+}
+
 function get() {
     if (!_settings) {
         _settings = read();
+        applyEnvironmentVariables(_settings);
     }
 
     return _settings;
@@ -529,6 +611,7 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
+// Legacy: can be removed after bridgeLegacy has been removed
 function whitelistDevice(ID) {
     const settings = get();
     if (!settings.whitelist) {
@@ -543,6 +626,16 @@ function whitelistDevice(ID) {
     write();
 }
 
+function blockDevice(ID) {
+    const settings = get();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
 function banDevice(ID) {
     const settings = get();
     if (!settings.ban) {
@@ -638,10 +731,18 @@ function removeGroup(groupIDorName) {
     write();
 }
 
-function changeDeviceOptions(IDorName, newOptions) {
-    const device = getDeviceThrowIfNotExists(IDorName);
+function changeEntityOptions(IDorName, newOptions) {
     const settings = get();
-    objectAssignDeep(settings.devices[device.ID], newOptions);
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
     write();
 }
 
@@ -672,20 +773,27 @@ module.exports = {
     getEntity,
     whitelistDevice,
     banDevice,
+    blockDevice,
     addDevice,
     removeDevice,
     addGroup,
     removeGroup,
     addDeviceToGroup,
     removeDeviceFromGroup,
-    changeDeviceOptions,
+    changeEntityOptions,
     changeFriendlyName,
 
     // For tests only
     _write: write,
     _reRead: () => {
-        _settings = read();
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        _settings = null;
+        get();
+        _settingsWithDefaults = null;
+        getWithDefaults();
+    },
+    _clear: () => {
+        _settings = null;
+        _settingsWithDefaults = null;
     },
     _getDefaults: () => defaults,
 };

lib/util/utils.js

@@ -1,5 +1,6 @@
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
+const humanizeDuration = require('humanize-duration');
+const data = require('./data');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -38,18 +39,10 @@ const endpointNames = [
     'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
-function flatten(arr) {
-    return arr.reduce((flat, toFlatten) => {
-        return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
-    }, []);
+function capitalize(s) {
+    return s[0].toUpperCase() + s.slice(1);
 }
 
-const forceEndDevice = flatten(
-    ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
-        .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
-        .map((definition) => definition.zigbeeModel));
-
 async function getZigbee2mqttVersion() {
     return new Promise((resolve, reject) => {
         const git = require('git-last-commit');
@@ -87,6 +80,10 @@ function formatDate(date, type) {
     case 'epoch':
         result = date;
         break;
+    case 'relative':
+        // https://github.com/EvanHahn/HumanizeDuration.js#options
+        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
+        break;
     default:
         throw new Error(`Unsupported type '${type}'`);
     }
@@ -114,14 +111,8 @@ function equalsPartial(object, expected) {
     return true;
 }
 
-function getObjectsProperty(objects, key, defaultValue) {
-    for (const object of objects) {
-        if (object.hasOwnProperty(key)) {
-            return object[key];
-        }
-    }
-
-    return defaultValue;
+function getObjectProperty(object, key, defaultValue) {
+    return object.hasOwnProperty(key) ? object[key] : defaultValue;
 }
 
 function getResponse(request, data, error) {
@@ -133,21 +124,67 @@ function getResponse(request, data, error) {
     return response;
 }
 
+function parseJSON(value, failedReturnValue) {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+function* getExternalConvertersDefinitions(settings) {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converterModule = moduleName;
+
+        if (moduleName.endsWith('.js')) {
+            converterModule = data.joinPath(moduleName.split('.')[0]);
+        }
+
+        const converter = require(converterModule);
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+function toSnakeCase(value) {
+    if (typeof value === 'object') {
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
     getZigbee2mqttVersion,
     objectHasProperties,
-    getObjectsProperty,
+    toSnakeCase,
+    getObjectProperty,
     getEndpointNames: () => endpointNames,
     isXiaomiDevice: (device) => {
         return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
             (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
     },
     isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
-    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelID),
-    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
     formatDate: (date, type) => formatDate(date, type),
     equalsPartial,
     getResponse,
+    capitalize,
+    parseJSON,
+    getExternalConvertersDefinitions,
 };

lib/util/yaml.js

@@ -1,23 +1,15 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
-const equals = require('fast-deep-equal');
+const equals = require('fast-deep-equal/es6');
 
 function read(file) {
     try {
         return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
     } catch (error) {
         if (error.name === 'YAMLException') {
-            error.message =
-                `\n\n\n` +
-                `\t====================================================================\n` +
-                `\tYour YAML file '${file}' is invalid\n` +
-                `\tUse e.g. https://jsonformatter.org/yaml-validator to find and fix the issue.\n` +
-                `\t====================================================================\n` +
-                `\n\n` +
-                error.message;
+            error.file = file;
         }
 
-
         throw error;
     }
 }

lib/zigbee.js

@@ -7,6 +7,7 @@ const utils = require('./util/utils');
 const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify');
 
 const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
@@ -42,7 +43,7 @@ class Zigbee extends events.EventEmitter {
 
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
-        logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
+        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
 
         if (herdsmanSettings.network.networkKey === 'GENERATE') {
             const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
@@ -67,18 +68,20 @@ class Zigbee extends events.EventEmitter {
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
 
         logger.info('zigbee-herdsman started');
-        logger.info(`Coordinator firmware version: '${JSON.stringify(await this.getCoordinatorVersion())}'`);
-        logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
+        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
         for (const device of this.getClients()) {
-            // If a whitelist is used, all other device will be removed from the network.
-            if (settings.get().whitelist.length > 0) {
-                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
                     device.removeFromNetwork();
                 }
-            } else if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
                 device.removeFromNetwork();
             }
         }
@@ -100,6 +103,10 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getCoordinatorVersion();
     }
 
+    async getNetworkParameters() {
+        return this.herdsman.getNetworkParameters();
+    }
+
     async reset(type) {
         await this.herdsman.reset(type);
     }
@@ -133,6 +140,10 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
     }
 
+    getDeviceByNetworkAddress(networkAddress) {
+        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
+    }
+
     getDevicesByType(type) {
         return this.herdsman.getDevicesByType(type);
     }
@@ -155,6 +166,7 @@ class Zigbee extends events.EventEmitter {
             `Wrong type '${typeof key}'`,
         );
 
+        const deviceOptions = settings.get().device_options;
         if (typeof key === 'string' || typeof key === 'number') {
             if (typeof key === 'number') {
                 key = key.toString();
@@ -209,12 +221,13 @@ class Zigbee extends events.EventEmitter {
                 }
 
                 return {
-                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
+                    type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
+                    name: entity.friendlyName, definition,
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
-                return {type: 'group', group, settings: entity, name: entity.friendlyName};
+                return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
             }
         } else if (key.constructor.name === 'Device') {
             const setting = settings.getEntity(key.ieeeAddr);
@@ -222,7 +235,7 @@ class Zigbee extends events.EventEmitter {
                 type: 'device',
                 device: key,
                 endpoint: key.endpoints[0],
-                settings: setting,
+                settings: {...deviceOptions, ...(setting || {})},
                 name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
                 definition: zigbeeHerdsmanConverters.findByDevice(key),
             };
@@ -231,7 +244,7 @@ class Zigbee extends events.EventEmitter {
             return {
                 type: 'group',
                 group: key,
-                settings: setting,
+                settings: {...deviceOptions, ...(setting || {})},
                 name: setting ? setting.friendlyName : key.groupID,
             };
         }
@@ -250,21 +263,23 @@ class Zigbee extends events.EventEmitter {
     }
 
     acceptJoiningDeviceHandler(ieeeAddr) {
-        // If set whitelist devices, all other device will be rejected to join the network
-        if (settings.get().whitelist.length > 0) {
-            if (settings.get().whitelist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
                 return true;
             } else {
-                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
                 return false;
             }
-        } else if (settings.get().ban.length > 0) {
-            if (settings.get().ban.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
                 return false;
             } else {
-                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
                 return true;
             }
         } else {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.14.0",
+  "version": "1.14.4",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -33,21 +33,23 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "*",
-    "debounce": "*",
-    "fast-deep-equal": "*",
-    "git-last-commit": "*",
-    "js-yaml": "*",
-    "mkdir-recursive": "*",
-    "moment": "*",
-    "mqtt": "*",
-    "object-assign-deep": "*",
-    "rimraf": "*",
-    "semver": "*",
-    "winston": "*",
-    "winston-syslog": "*",
-    "zigbee-herdsman": "0.12.94",
-    "zigbee-herdsman-converters": "12.0.110"
+    "ajv": "^6.12.4",
+    "debounce": "^1.2.0",
+    "fast-deep-equal": "^3.1.3",
+    "git-last-commit": "^1.0.0",
+    "humanize-duration": "^3.23.1",
+    "js-yaml": "^3.14.0",
+    "json-stable-stringify": "^1.0.1",
+    "mkdir-recursive": "^0.4.0",
+    "moment": "^2.27.0",
+    "mqtt": "^4.2.1",
+    "object-assign-deep": "^0.4.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.2",
+    "winston": "^3.3.3",
+    "winston-syslog": "^2.4.4",
+    "zigbee-herdsman": "0.12.127",
+    "zigbee-herdsman-converters": "12.0.178"
   },
   "devDependencies": {
     "eslint": "*",

scripts/install.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 function zigbee2mqtt-show-short-info {
-  echo "Setup for Zigbee2mqtt bridge."
+  echo "Setup for Zigbee2MQTT bridge."
 }
 
 function zigbee2mqtt-show-long-info {
-  echo "This script installs the Zigbee2mqtt bridge"
+  echo "This script installs the Zigbee2MQTT bridge"
 }
 
 function zigbee2mqtt-show-copyright-info {
@@ -20,7 +20,7 @@ if [ -z "${node}" ]; then #Installing NodeJS if not already installed.
   apt install -y nodejs
 fi
 
-echo "Cloning Zigbee2mqtt git repository"
+echo "Cloning Zigbee2MQTT git repository"
 git clone https://github.com/Koenkk/zigbee2mqtt.git /opt/zigbee2mqtt
 chown -R pi:pi /opt/zigbee2mqtt
 

test/assets/mock-external-converter-multiple.js

@@ -0,0 +1,20 @@
+const homeassistantSwitch = {
+    type: 'switch',
+    object_id: 'switch',
+    discovery_payload: {
+        payload_off: 'OFF',
+        payload_on: 'ON',
+        value_template: '{{ value_json.state }}',
+        command_topic: true,
+    },
+};
+
+const mockDevices = [{
+    mock: 1,
+    model: 'external_converters_device',
+    homeassistant: [homeassistantSwitch],
+}, {
+    mock: 2
+}];
+
+module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -0,0 +1,5 @@
+const mockDevice = {
+    mock: true
+};
+
+module.exports = mockDevice;

test/availability.test.js

@@ -197,7 +197,21 @@ describe('Availability', () => {
         expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
     });
 
-    it('Should not ping devices blacklisted by friendly name', async () => {
+    it('Should not ping devices on blocklist by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
         await controller.stop();
@@ -211,9 +225,9 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not ping devices blacklisted by IEEE address', async () => {
+    it('Should not ping devices on blocklist by IEEE address', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -238,7 +252,21 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
         await controller.stop();
@@ -252,9 +280,9 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
+        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
         await controller.stop();
         await flushPromises();
         controller = new Controller();
@@ -266,10 +294,10 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         getExtension().state[device.ieeeAddr] = false;
-        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
+        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
         await controller.stop();
         await flushPromises();
         controller = new Controller();

test/bind.test.js

@@ -5,6 +5,7 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
+const stringify = require('json-stable-stringify');
 
 describe('Bind', () => {
     let controller;
@@ -14,6 +15,7 @@ describe('Bind', () => {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
+            endpoint.bind = jest.fn();
             endpoint.bind.mockClear();
             endpoint.unbind.mockClear();
         }
@@ -23,6 +25,7 @@ describe('Bind', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -31,10 +34,228 @@ describe('Bind', () => {
     });
 
     it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
     });
 
     it('Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind only specifief clusters', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color', clusters: ["genOnOff"]}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'button'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'Coordinator'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: '1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementation(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: target}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when source not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when target not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -54,7 +275,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log error when there is nothing to bind', async () => {
+    it('Legacy api: Should log error when there is nothing to bind', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
@@ -65,7 +286,7 @@ describe('Bind', () => {
         expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
     });
 
-    it('Should unbind', async () => {
+    it('Legacy api: Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -85,7 +306,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should unbind coordinator', async () => {
+    it('Legacy api: Should unbind coordinator', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -106,7 +327,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to groups', async () => {
+    it('Legacy api: Should bind to groups', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -126,7 +347,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to group by number', async () => {
+    it('Legacy api: Should bind to group by number', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -146,7 +367,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log when bind fails', async () => {
+    it('Legacy api: Should log when bind fails', async () => {
         logger.error.mockClear();
         const device = zigbeeHerdsman.devices.remote;
         const endpoint = device.getEndpoint(1);
@@ -154,12 +375,11 @@ describe('Bind', () => {
         endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
         MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
     });
 
-    it('Should bind from non default endpoints', async () => {
+    it('Legacy api: Should bind from non default endpoints', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
         const endpoint = device.getEndpoint(2);
@@ -170,7 +390,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should bind to default endpoint returned by endpoints()', async () => {
+    it('Legacy api: Should bind to default endpoint returned by endpoints()', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
         const endpoint = device.getEndpoint(2);
@@ -181,7 +401,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should unbind from default_bind_group', async () => {
+    it('Legacy api: Should unbind from default_bind_group', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = 'default_bind_group';
         const endpoint = device.getEndpoint(1);

test/bridge.test.js

@@ -5,17 +5,20 @@ const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
-const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const stringify = require('json-stable-stringify');
 
-const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+const {coordinator, bulb, unsupported, WXKG11LM, remote} = zigbeeHerdsman.devices;
 zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(WXKG11LM.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(remote.ieeeAddr);
 
 describe('Bridge', () => {
     let controller;
 
     beforeEach(async () => {
+        MQTT.mock.reconnecting = false;
         data.writeDefaultConfiguration();
         settings._reRead();
         settings.set(['advanced', 'legacy_api'], false);
@@ -23,7 +26,11 @@ describe('Bridge', () => {
         data.writeDefaultState();
         logger.info.mockClear();
         logger.warn.mockClear();
+        logger.setTransportsEnabled(false);
         MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        device.removeFromDatabase.mockClear();
+        device.removeFromNetwork.mockClear();
         controller = new Controller();
         await controller.start();
         await flushPromises();
@@ -33,7 +40,7 @@ describe('Bridge', () => {
         const version = await require('../lib/util/utils').getZigbee2mqttVersion();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/info',
-          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"logLevel":"info","permitJoin":false}),
+          stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"network":{"channel":15,"pan_id":5674,"extended_pan_id":[0,11,22]},"log_level":"info","permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -42,16 +49,43 @@ describe('Bridge', () => {
     it('Should publish devices on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/devices',
-          JSON.stringify([{"ieeeAddress":"0x000b57fffec6a5b2","type":"Router","networkAddress":40369,"supported":true,"friendlyName":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"powerSource":"Mains (single phase)","dateCode":null,"interviewing":false,"interviewCompleted":true},{"ieeeAddress":"0x0017880104e45518","type":"EndDevice","networkAddress":6536,"supported":false,"friendlyName":"0x0017880104e45518","definition":null,"powerSource":"Battery","dateCode":null,"interviewing":false,"interviewCompleted":true}]),
+          stringify([{"date_code":null,"definition":null,"endpoints":{"1":{"bindings":[],"clusters":{"input":[],"output":[]}}},"friendly_name":"Coordinator","ieee_address":"0x00124b00120144ae","interview_completed":false,"interviewing":false,"network_address":0,"power_source":null,"supported":false,"type":"Coordinator"},{"date_code":null,"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"on/off, brightness, color temperature","vendor":"IKEA"},"endpoints":{"1":{"bindings":[],"clusters":{"input":["genBasic","genScenes","genOnOff","genLevelCtrl","lightingColorCtrl"],"output":["genScenes","genOta"]}}},"friendly_name":"bulb","ieee_address":"0x000b57fffec6a5b2","interview_completed":true,"interviewing":false,"network_address":40369,"power_source":"Mains (single phase)","supported":true,"type":"Router"},{"date_code":null,"definition":{"description":"Hue dimmer switch","model":"324131092621","supports":"on/off, brightness, up/down/hold/release, click count","vendor":"Philips"},"endpoints":{"1":{"bindings":[{"cluster":"genLevelCtrl","target":{"endpoint":1,"ieee_address":"0x000b57fffec6a5b3","type":"endpoint"}},{"cluster":"genOnOff","target":{"id":1,"type":"group"}}],"clusters":{"input":["genBasic"],"output":["genBasic","genOnOff","genLevelCtrl","genScenes"]}},"2":{"bindings":[],"clusters":{"input":["genBasic"],"output":["genOta","genOnOff"]}}},"friendly_name":"remote","ieee_address":"0x0017880104e45517","interview_completed":true,"interviewing":false,"network_address":6535,"power_source":"Battery","supported":true,"type":"EndDevice"},{"date_code":null,"definition":null,"endpoints":{"1":{"bindings":[],"clusters":{"input":["genBasic"],"output":["genBasic","genOnOff","genLevelCtrl","genScenes"]}}},"friendly_name":"0x0017880104e45518","ieee_address":"0x0017880104e45518","interview_completed":true,"interviewing":false,"network_address":6536,"power_source":"Battery","supported":false,"type":"EndDevice"},{"date_code":null,"definition":{"description":"Aqara wireless switch","model":"WXKG11LM","supports":"single, double click (and triple, quadruple, hold, release depending on model)","vendor":"Xiaomi"},"endpoints":{"1":{"bindings":[],"clusters":{"input":["genBasic"],"output":["genBasic","genOnOff","genLevelCtrl","genScenes"]}}},"friendly_name":"button","ieee_address":"0x0017880104e45520","interview_completed":true,"interviewing":false,"network_address":6537,"power_source":"Battery","supported":true,"type":"EndDevice"}]),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
     });
 
-    it('Should publish devices on startup', async () => {
+    it('Should log to MQTT', async () => {
+        logger.setTransportsEnabled(true);
+        MQTT.publish.mockClear();
+        logger.info.mockClear();
+        logger.info("this is a test");
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/logging',
+            stringify({message: 'this is a test', level: 'info'}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(logger.info).toHaveBeenCalledTimes(1);
+    });
+
+    it('Shouldnt log to MQTT when not connected', async () => {
+        logger.setTransportsEnabled(true);
+        MQTT.mock.reconnecting = true;
+        MQTT.publish.mockClear();
+        logger.info.mockClear();
+        logger.error.mockClear();
+        logger.info("this is a test");
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish groups on startup', async () => {
+        logger.setTransportsEnabled(true);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/groups',
-          JSON.stringify([{"ID":1,"friendlyName":"group_1","members":[]},{"ID":15071,"friendlyName":"group_tradfri_remote","members":[]},{"ID":99,"friendlyName":99,"members":[]},{"ID":11,"friendlyName":"group_with_tradfri","members":[]},{"ID":2,"friendlyName":"group_2","members":[]}]),
+          stringify([{"friendly_name":"group_1","id":1,"members":[]},{"friendly_name":"group_tradfri_remote","id":15071,"members":[]},{"friendly_name":99,"id":99,"members":[]},{"friendly_name":"group_with_tradfri","id":11,"members":[]},{"friendly_name":"thermostat_group","id":12,"members":[]},{"friendly_name":"group_2","id":2,"members":[]}]),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -64,7 +98,20 @@ describe('Bridge', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/event',
-          JSON.stringify({"type":"deviceJoined","data":{"friendlyName":"bulb","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          stringify({"type":"device_joined","data":{"friendly_name":"bulb","ieee_address":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device announces', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          stringify({"type":"device_announce","data":{"friendly_name":"bulb","ieee_address":"0x000b57fffec6a5b2"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -77,7 +124,7 @@ describe('Bridge', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/event',
-          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"started","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"started","ieee_address":"0x000b57fffec6a5b2"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -90,7 +137,7 @@ describe('Bridge', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/event',
-          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"failed","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"failed","ieee_address":"0x000b57fffec6a5b2"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -110,13 +157,13 @@ describe('Bridge', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/event',
-            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"successful","ieeeAddress":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            stringify({"type":"device_interview","data":{"friendly_name":"bulb","status":"successful","ieee_address":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
             { retain: false, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
           'zigbee2mqtt/bridge/event',
-            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"0x0017880104e45518","status":"successful","ieeeAddress":"0x0017880104e45518","supported":false,"definition":null}}),
+            stringify({"type":"device_interview","data":{"friendly_name":"0x0017880104e45518","status":"successful","ieee_address":"0x0017880104e45518","supported":false,"definition":null}}),
             { retain: false, qos: 0 },
             expect.any(Function)
         );
@@ -135,7 +182,7 @@ describe('Bridge', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/event',
-          JSON.stringify({"type":"deviceLeave","data":{"ieeeAddress":"0x000b57fffec6a5b2"}}),
+          stringify({"type":"device_leave","data":{"ieee_address":"0x000b57fffec6a5b2"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -150,38 +197,38 @@ describe('Bridge', () => {
     it('Should allow permit join', async () => {
         zigbeeHerdsman.permitJoin.mockClear();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', 'true');
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', 'true');
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bridge/response/permitJoin',
-            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            'zigbee2mqtt/bridge/response/permit_join',
+            stringify({"data":{"value":true},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
 
         zigbeeHerdsman.permitJoin.mockClear();
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', stringify({"value": false}));
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bridge/response/permitJoin',
-            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            'zigbee2mqtt/bridge/response/permit_join',
+            stringify({"data":{"value":false},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
     it('Should put transaction in response when request is done with transaction', async () => {
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false, "transaction": 22}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', stringify({"value": false, "transaction": 22}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bridge/response/permitJoin',
-            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            'zigbee2mqtt/bridge/response/permit_join',
+            stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -189,20 +236,474 @@ describe('Bridge', () => {
     it('Should put error in response when request fails', async () => {
         zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', stringify({"value": false}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bridge/response/permitJoin',
-            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            'zigbee2mqtt/bridge/response/permit_join',
+            stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when format is incorrect', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permit_join', stringify({"value_not_good": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permit_join',
+            stringify({"data":{},"status":"error","error": "No value given"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
 
     it('Coverage satisfaction', async () => {
         MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', stringify({"value": false}));
         const device = zigbeeHerdsman.devices.bulb;
         await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
         await flushPromises();
     });
+
+    it('Should allow a healthcheck', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/health_check', '');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/health_check',
+            stringify({"data":{"healthy": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to remove device by string', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb');
+        await flushPromises();
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual([]);
+    });
+
+    it('Should allow to remove device by object ID', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "bulb"}));
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(device.removeFromDatabase).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{"id": "bulb","block":false,"force":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "bulb", force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(device.removeFromNetwork).not.toHaveBeenCalled();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{"id": "bulb","block":false,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to block device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "bulb", block: true, force: true}));
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{"id": "bulb","block":true,"force":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().blocklist).toStrictEqual(["0x000b57fffec6a5b2"]);
+    });
+
+    it('Should allow to remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', 'group_1');
+        await flushPromises();
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            stringify({"data":{"id": "group_1", "force": false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove group', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/remove', stringify({id: "group_1", force: true}));
+        await flushPromises();
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/remove',
+            stringify({"data":{"id": "group_1", "force": true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on removing non-existing device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "non-existing-device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{},"status":"error","error":"Device 'non-existing-device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when remove device fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('device timeout')})
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', stringify({id: "bulb"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/remove',
+            stringify({"data":{},"status":"error","error":"Failed to remove device 'bulb' (block: false, force: false) (Error: device timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow rename group', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/rename', stringify({from: 'group_1', to: 'group_new_name'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toBeNull();
+        expect(settings.getGroup('group_new_name')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_new_name", "friendlyName": "group_new_name", "optimistic": true, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/rename',
+            stringify({"data":{"from":"group_1","to":"group_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device rename payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from_bla: 'bulb', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on non-existing device rename', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb_not_existing', to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            stringify({"data":{},"status":"error","error":"Device 'bulb_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toBeNull();
+        expect(settings.getDevice('bulb_new_name')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb_new_name", "friendlyName": "bulb_new_name", "retain": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/devices', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            stringify({"data":{"from":"bulb","to":"bulb_new_name"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when renaming last joined device but none has joined', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({last: true, to: 'bulb_new_name'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/rename',
+            stringify({"data":{},"status":"error","error":"No device has joined since start"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change device options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', stringify({options: {retain: false, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb')).toStrictEqual({"ID": "0x000b57fffec6a5b2", "friendly_name": "bulb", "friendlyName": "bulb", "retain": false, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            stringify({"data":{"from":{"retain": true},"to":{"retain": false,"transition":1}, "id":"bulb"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow change group options', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": false, "friendlyName": "group_1", "optimistic": true});
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/options', stringify({options: {retain: true, transition: 1}, id: 'group_1'}));
+        await flushPromises();
+        expect(settings.getGroup('group_1')).toStrictEqual({"ID": 1, "devices": [], "friendly_name": "group_1", "retain": true, "friendlyName": "group_1", "optimistic": true, "transition": 1});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/options',
+            stringify({"data":{"from":{"optimistic": true,"retain": false},"to":{"optimistic": true,"retain": true,"transition":1}, "id":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error on invalid device change options payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/options', stringify({options_: {retain: true, transition: 1}, id: 'bulb'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/options',
+            stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group by string', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', 'group_193');
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 3, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            stringify({"data":{"friendly_name":"group_193","id": 3},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to add group with ID', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', stringify({friendly_name: "group_193", id: 9}));
+        await flushPromises();
+        expect(settings.getGroup('group_193')).toStrictEqual({"ID": 9, "devices": [], "friendly_name": "group_193", "friendlyName": "group_193", "optimistic": true});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            stringify({"data":{"friendly_name":"group_193","id": 9},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should throw error when add with invalid payload', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/add', stringify({friendly_name9: "group_193"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/add',
+            stringify({"data":{},"status":"error","error":"Invalid payload"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to enable/disable Home Assistant extension', async () => {
+        // Test if disabled intially
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        expect(settings.get().homeassistant).toBeFalsy();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/button/action', 'single', {retain: false, qos: 0}, expect.any(Function));
+
+        // Disable when already disabled should go OK
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/homeassistant', stringify({value: false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/homeassistant',
+            stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().homeassistant).toBeFalsy();
+
+        // Enable
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/homeassistant', stringify({value: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/homeassistant',
+            stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().homeassistant).toBeTruthy();
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button/action', 'single', {retain: false, qos: 0}, expect.any(Function));
+
+        // Disable
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/homeassistant', stringify({value: false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/homeassistant',
+            stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(settings.get().homeassistant).toBeFalsy();
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/button/action', 'single', {retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should fail to set Home Assistant when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/homeassistant', 'invalid_one');
+        await flushPromises();
+        expect(settings.get().homeassistant).toBeFalsy();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/homeassistant',
+            stringify({"data":{},"status":"error","error":"'invalid_one' is not an allowed value, allowed: true,false"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            stringify({"data":{"value":"ISO_8601"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/last_seen', 'invalid_one');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/last_seen',
+            stringify({"data":{},"status":"error","error":"'invalid_one' is not an allowed value, allowed: disable,ISO_8601,epoch,ISO_8601_local"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set last_seen when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/elapsed', 'not_valid');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/elapsed',
+            stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: true,false"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to set log level', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            stringify({"data":{"value":'debug'},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail to set log level when invalid type', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/config/log_level', 'not_valid');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/config/log_level',
+            stringify({"data":{},"status":"error","error":"'not_valid' is not an allowed value, allowed: error,warn,info,debug"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (succeeds)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            stringify({"data":{},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should allow to touchlink factory reset (fails)', async () => {
+        MQTT.publish.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/request/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/touchlink/factory_reset',
+            stringify({"data":{},"status":"error","error":"Failed to factory reset device through Touchlink"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
 });

test/configure.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const stringify = require('json-stable-stringify');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
@@ -68,6 +69,16 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
+    it('Should reconfigure reporting on reportingDisabled event', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        mockClear(device);
+        expectRemoteNotConfigured();
+        controller.eventBus.emit('reportingDisabled', {device})
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
     it('Should not configure twice', async () => {
         expectRemoteConfigured();
         const device = zigbeeHerdsman.devices.remote;
@@ -90,8 +101,52 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Should allow to reconfigure manually', async () => {
+    it('Should allow to configure via MQTT', async () => {
         expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "not_existing_device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when configure fails', async () => {
+        zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "remote"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device has no configure', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb", transaction: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should allow to reconfigure manually', async () => {
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
@@ -99,13 +154,13 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Shouldnt manually reconfigure when device does not exist', async () => {
+    it('Legacy api: Shouldnt manually reconfigure when device does not exist', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
         await flushPromises();
         expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
     });
 
-    it('Should skip reconfigure when device does not require this', async () => {
+    it('Legacy api: Should skip reconfigure when device does not require this', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
         await flushPromises();
         expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)

test/controller.test.js

@@ -6,6 +6,7 @@ const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify');
 const flushPromises = () => new Promise(setImmediate);
 const tmp = require('tmp');
 const mocksClear = [
@@ -30,7 +31,7 @@ describe('Controller', () => {
 
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
@@ -42,8 +43,15 @@ describe('Controller', () => {
         expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',{ retain: true, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Start controller when permit join fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error("failed!")});
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
     });
 
     it('Start controller with specific MQTT settings', async () => {
@@ -97,8 +105,17 @@ describe('Controller', () => {
         data.writeDefaultState();
         await controller.start();
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Start controller should not publish cached states when disabled', async () => {
+        settings.set(['advanced', 'cache_state_send_on_startup'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        const publishedTopics = MQTT.publish.mock.calls.map(m => m[0]);
+        expect(publishedTopics).toEqual(expect.not.arrayContaining(["zigbee2mqtt/bulb", "zigbee2mqtt/remote"]));
     });
 
     it('Start controller should not publish cached states when cache_state is false', async () => {
@@ -132,7 +149,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(2);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
-        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"state\":\"ON\",\"brightness\":50,\"color_temp\":370,\"linkquality\":99,\"color\":{\"r\":100,\"g\":50,\"b\":10},\"dummy\":{\"1\":\"yes\",\"2\":\"no\"}}");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"brightness\":50,\"color\":{\"b\":10,\"g\":50,\"r\":100},\"color_temp\":370,\"dummy\":{\"1\":\"yes\",\"2\":\"no\"},\"linkquality\":99,\"state\":\"ON\"}");
         controller.mqtt.client.reconnecting = false;
     });
 
@@ -143,6 +160,14 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
+    it('Should remove device not on passlist on startup', async () => {
+        settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
     it('Should remove non whitelisted devices on startup', async () => {
         settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -151,6 +176,14 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
     });
 
+    it('Should remove device on blocklist on startup', async () => {
+        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
     it('Should remove banned devices on startup', async () => {
         settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -172,15 +205,6 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
     });
 
-    it('Refuse to start when configuration.yaml is invalid', async () => {
-        settings.set(['permit_join'], 'invalid');
-        await controller.start();
-        expect(logger.error).toHaveBeenCalledWith('Refusing to start, configuration.yaml is not valid, found the following errors:');
-        expect(logger.error).toHaveBeenCalledWith('\t - permit_join should be boolean');
-        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
-    });
-
     it('Start controller with disable_led', async () => {
         settings.set(['serial', 'disable_led'], true);
         await controller.start();
@@ -273,51 +297,51 @@ describe('Controller', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('acceptJoiningDeviceHandler reject banned device', async () => {
+    it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+    it('acceptJoiningDeviceHandler accept device not on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], ['123']);
+        settings.set(['blocklist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler accept device on passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler reject device not in passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], ['123']);
+        settings.set(['passlist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+    it('acceptJoiningDeviceHandler should prefer passlist above blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+    it('acceptJoiningDeviceHandler accept when not on blocklist and passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
@@ -331,7 +355,7 @@ describe('Controller', () => {
         zigbeeHerdsman.events.deviceJoined(payload);
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee deviceInterview started', async () => {
@@ -340,7 +364,7 @@ describe('Controller', () => {
         const payload = {device, status: 'started'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview failed', async () => {
@@ -349,7 +373,7 @@ describe('Controller', () => {
         const payload = {device, status: 'failed'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful supported', async () => {
@@ -358,7 +382,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful not supported', async () => {
@@ -367,7 +391,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee event device announce', async () => {
@@ -377,7 +401,7 @@ describe('Controller', () => {
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
         expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and settings)', async () => {
@@ -389,7 +413,7 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and NOT settings)', async () => {
@@ -400,21 +424,23 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Publish entity state attribute output', async () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute');
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await controller.publishEntityState('bulb', {dummy: {1: 'yes', 2: 'no'}, color: {r: 100, g: 50, b: 10}, state: 'ON', test: undefined, test1: null, color_temp: 370, brightness: 50});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", 'yes', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", "yes", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", "no", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test1", '', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test", '', {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state attribute_json output', async () => {
@@ -428,7 +454,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
 
@@ -442,7 +468,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state attribute_json output filtered (device_options)', async () => {
@@ -455,7 +481,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state with device information', async () => {
@@ -464,12 +490,12 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}), {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
         await controller.publishEntityState('unsupported2', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state without retain', async () => {
@@ -478,7 +504,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state with retain', async () => {
@@ -487,7 +513,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Should publish entity state with expiring retention', async () => {
@@ -498,7 +524,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
     });
 
     it('Publish entity state no empty messages', async () => {
@@ -510,6 +536,21 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should allow to disable state persistency', async () => {
+        settings.set(['advanced', 'cache_state_persistent'], false);
+        data.removeState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        await controller.stop();
+        expect(data.stateExists()).toBeFalsy();
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();
@@ -519,8 +560,8 @@ describe('Controller', () => {
         await controller.publishEntityState('bulb', {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"state\":\"ON\"}", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"brightness\":200}", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish should not do anything for unknown entity', async () => {

test/externalConverters.test.js

@@ -0,0 +1,115 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const path = require('path');
+const fs = require('fs');
+
+zigbeeHerdsmanConverters.addDeviceDefinition = jest.fn();
+
+const mocksClear = [zigbeeHerdsmanConverters.addDeviceDefinition, zigbeeHerdsman.permitJoin,
+    mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+jest.mock(
+    'mock-external-converter-module', () => {
+        return {
+            mock: true
+        };
+    }, {
+        virtual: true
+    });
+
+jest.mock(
+    'mock-multiple-external-converter-module', () => {
+        return [{
+            mock: 1
+        }, {
+            mock: 2
+        }];
+    }, {
+        virtual: true
+    });
+
+describe('Loads external converters', () => {
+    let controller;
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        await flushPromises();
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        data.writeEmptyState();
+        settings._reRead();
+    });
+
+    it('Does not load external converters', async () => {
+        settings.set(['external_converters'], []);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
+    });
+
+    it('Loads external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1,
+            model: 'external_converters_device',
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+
+    it('Loads external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+});

test/group.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
@@ -16,6 +17,7 @@ describe('Groups', () => {
 
     beforeEach(() => {
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
@@ -45,6 +47,19 @@ describe('Groups', () => {
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
+    it('Apply group updates remove handle fail', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        endpoint.removeFromGroup.mockImplementationOnce(() => {throw new Error("failed!")});
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        logger.error.mockClear();
+        await controller.start();
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
+    });
+
     it('Move to non existing group', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
@@ -81,7 +96,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
-    it('Add to group via MQTT', async () => {
+    it('Legacy api: Add to group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -93,10 +108,10 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Add to group with slashes via MQTT', async () => {
+    it('Legacy api: Add to group with slashes via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups["group/with/slashes"];
@@ -108,7 +123,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
 
         // Test if subscribed to topics with slashes
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
@@ -117,7 +132,7 @@ describe('Groups', () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
-    it('Add to group via MQTT with postfix', async () => {
+    it('Legacy api: Add to group via MQTT with postfix', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -130,7 +145,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+    it('Legacy api: Add to group via MQTT with postfix shouldnt add it twice', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
@@ -145,7 +160,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove from group via MQTT', async () => {
+    it('Legacy api: Remove from group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -157,10 +172,10 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+    it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -174,7 +189,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
     });
 
-    it('Remove from group via MQTT with postfix variant 1', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 1', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -188,7 +203,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 2', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 2', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -202,7 +217,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 3', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 3', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -216,7 +231,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group all', async () => {
+    it('Legacy api: Remove from group all', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
@@ -228,7 +243,7 @@ describe('Groups', () => {
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Remove from group all deprecated', async () => {
@@ -245,7 +260,7 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Log when adding to non-existing group', async () => {
+    it('Legacy api: Log when adding to non-existing group', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -254,7 +269,7 @@ describe('Groups', () => {
         expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
     });
 
-    it('Log when adding to non-existing device', async () => {
+    it('Legacy api: Log when adding to non-existing device', async () => {
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
@@ -278,8 +293,8 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","linkquality":10}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -292,11 +307,11 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state, filtered', async () => {
@@ -309,11 +324,11 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","brightness":100}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Shouldnt publish group state change when a group is not optimistic', async () => {
@@ -331,7 +346,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","linkquality":10}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of another group with shared device when a group changes its state', async () => {
@@ -348,12 +363,12 @@ describe('Groups', () => {
         await flushPromises();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via device', async () => {
@@ -370,14 +385,14 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -395,15 +410,15 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -420,17 +435,17 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF","brightness":0}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -447,16 +462,278 @@ describe('Groups', () => {
         await controller.start();
         await flushPromises();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
         controller.state.state = {};
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF","brightness":0}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove_all',
+            stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/homeassistant.test.js

@@ -1,15 +1,19 @@
 const data = require('./stub/data');
 const settings = require('../lib/util/settings');
+const stringify = require('json-stable-stringify');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const flushPromises = () => new Promise(setImmediate);
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
+const fs = require('fs');
+const path = require('path');
+const HomeAssistant = require('../lib/extension/homeassistant');
 
 describe('HomeAssistant extension', () => {
     beforeEach(async () => {
         this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
-        this.version = `Zigbee2mqtt ${this.version.version}`;
+        this.version = `Zigbee2MQTT ${this.version.version}`;
         jest.useRealTimers();
         data.writeDefaultConfiguration();
         settings._reRead();
@@ -20,7 +24,6 @@ describe('HomeAssistant extension', () => {
 
     it('Should have mapping for all devices supported by zigbee-herdsman-converters', () => {
         const missing = [];
-        const HomeAssistant = require('../lib/extension/homeassistant');
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
@@ -34,7 +37,6 @@ describe('HomeAssistant extension', () => {
 
     it('Should not have duplicate type/object_ids in a mapping', () => {
         const duplicated = [];
-        const HomeAssistant = require('../lib/extension/homeassistant');
         const ha = new HomeAssistant(null, null, null, null, {on: () => {}});
 
         require('zigbee-herdsman-converters').devices.forEach((d) => {
@@ -80,7 +82,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -105,7 +107,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -130,7 +132,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/pressure/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -155,7 +157,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/battery/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -180,7 +182,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/linkquality/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -221,7 +223,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -246,7 +248,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -271,7 +273,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/pressure/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -329,7 +331,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -355,7 +357,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/humidity/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -418,7 +420,67 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/fan/0x0017880104e45548/fan/config',
-            JSON.stringify(payload),
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+    });
+
+    it('Should discover thermostat devices', async () => {
+        controller = new Controller(false);
+        await controller.start();
+
+        let payload;
+        await flushPromises();
+
+        payload = {
+            "action_template":"{% set values = {'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'} %}{{ values[value_json.running_state] }}",
+            "action_topic":"zigbee2mqtt/TS0601_thermostat",
+            "availability_topic":"zigbee2mqtt/bridge/state",
+            "current_temperature_template":"{{ value_json.local_temperature }}",
+            "current_temperature_topic":"zigbee2mqtt/TS0601_thermostat",
+            "device":{
+               "identifiers":[
+                  "zigbee2mqtt_0x0017882104a44559"
+               ],
+               "manufacturer":"TuYa",
+               "model":"Radiator valve with thermostat (TS0601_thermostat)",
+               "name":"TS0601_thermostat",
+               "sw_version":  this.version,
+            },
+            "hold_command_topic":"zigbee2mqtt/TS0601_thermostat/set/preset",
+            "hold_modes":[
+               "schedule",
+               "manual",
+               "away",
+               "boost",
+               "complex",
+               "comfort",
+               "eco"
+            ],
+            "hold_state_template":"{{ value_json.preset }}",
+            "hold_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "json_attributes_topic":"zigbee2mqtt/TS0601_thermostat",
+            "max_temp":"30",
+            "min_temp":"5",
+            "mode_command_topic":"zigbee2mqtt/TS0601_thermostat/set/system_mode",
+            "mode_state_template":"{{ value_json.system_mode }}",
+            "mode_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "modes":[
+               "auto"
+            ],
+            "name":"TS0601_thermostat_climate",
+            "temp_step":0.5,
+            "temperature_command_topic":"zigbee2mqtt/TS0601_thermostat/set/current_heating_setpoint",
+            "temperature_state_template":"{{ value_json.current_heating_setpoint }}",
+            "temperature_state_topic":"zigbee2mqtt/TS0601_thermostat",
+            "temperature_unit":"C",
+            "unique_id":"0x0017882104a44559_climate_zigbee2mqtt"
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/climate/0x0017882104a44559/climate/config',
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -432,18 +494,18 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
 
         payload = {
-            command_topic: 'zigbee2mqtt/smart_vent/set',
-            position_topic: 'zigbee2mqtt/smart_vent',
-            set_position_topic: 'zigbee2mqtt/smart_vent/set',
+            command_topic: 'zigbee2mqtt/smart vent/set',
+            position_topic: 'zigbee2mqtt/smart vent',
+            set_position_topic: 'zigbee2mqtt/smart vent/set',
             set_position_template: '{ "position": {{ position }} }',
             value_template: '{{ value_json.position }}',
-            json_attributes_topic: 'zigbee2mqtt/smart_vent',
-            name: 'smart_vent_cover',
+            json_attributes_topic: 'zigbee2mqtt/smart vent',
+            name: 'smart vent cover',
             unique_id: '0x0017880104e45551_cover_zigbee2mqtt',
             device:
             {
                 identifiers: [ 'zigbee2mqtt_0x0017880104e45551' ],
-                name: 'smart_vent',
+                name: 'smart vent',
                 sw_version: this.version,
                 model: 'Smart vent (SV01)',
                 manufacturer: 'Keen Home'
@@ -453,7 +515,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/cover/0x0017880104e45551/cover/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -487,7 +549,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'my_custom_discovery_topic/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -552,7 +614,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payloadHA),
+            stringify(payloadHA),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -572,12 +634,39 @@ describe('HomeAssistant extension', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
     });
 
+    it('Should send all status when home assistant comes online (default topic)', async () => {
+        jest.useFakeTimers();
+        data.writeDefaultState();
+        controller = new Controller();
+        await controller.start();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('homeassistant/status');
+        await flushPromises();
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/status', 'online');
+        await flushPromises();
+        jest.runOnlyPendingTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote',
+            stringify({"brightness":255,"update_available":false}),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
     it('Should send all status when home assistant comes online', async () => {
         jest.useFakeTimers();
         data.writeDefaultState();
         controller = new Controller();
         await controller.start();
         await flushPromises();
+        expect(MQTT.subscribe).toHaveBeenCalledWith('hass/status');
         MQTT.publish.mockClear();
         await MQTT.events.message('hass/status', 'online');
         await flushPromises();
@@ -585,13 +674,13 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb',
-            '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}',
+            stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/remote',
-            '{"brightness":255,"update_available":false}',
+            stringify({"brightness":255,"update_available":false}),
             { retain: true, qos: 0 },
             expect.any(Function)
         );
@@ -653,7 +742,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -737,7 +826,14 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/sensor/0x0017880104e45522/temperature/config',
-            JSON.stringify(payload),
+            stringify(payload),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/sensor/0x0017880104e45522/temperature/config',
+            null,
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -769,7 +865,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config',
-            JSON.stringify(payload),
+            stringify(payload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
@@ -792,7 +888,31 @@ describe('HomeAssistant extension', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
 
-        const discoverPayload = {
+        const discoverPayloadAction = {
+            "automation_type":"trigger",
+            "type":"action",
+            "subtype":"single",
+            "payload":"single",
+            "topic":"zigbee2mqtt/button/action",
+            "device":{
+                "identifiers":[
+                    "zigbee2mqtt_0x0017880104e45520"
+                ],
+                "name":"button",
+                "sw_version": this.version,
+                "model":"Aqara wireless switch (WXKG11LM)",
+                "manufacturer":"Xiaomi"
+            }
+        };
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/action_single/config',
+            stringify(discoverPayloadAction),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        const discoverPayloadClick = {
             "automation_type":"trigger",
             "type":"click",
             "subtype":"single",
@@ -811,11 +931,18 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'homeassistant/device_automation/0x0017880104e45520/click_single/config',
-            JSON.stringify(discoverPayload),
+            stringify(discoverPayloadClick),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
 
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/action',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button/click',
             'single',
@@ -825,14 +952,21 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            JSON.stringify({click: "single", linkquality: 10}),
+            stringify({action: "single", linkquality: 10, click: "single"}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            JSON.stringify({linkquality: 10, click: ""}),
+            stringify({linkquality: 10, action: ""}),
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button',
+            stringify({linkquality: 10, click: ""}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -842,12 +976,26 @@ describe('HomeAssistant extension', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).not.toHaveBeenCalledWith(
-            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
-            JSON.stringify(discoverPayload),
+            'homeassistant/device_automation/0x0017880104e45520/action_single/config',
+            stringify(discoverPayloadAction),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
 
+        expect(MQTT.publish).not.toHaveBeenCalledWith(
+            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
+            stringify(discoverPayloadClick),
+            { retain: true, qos: 0 },
+            expect.any(Function),
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button/action',
+            'single',
+            { retain: false, qos: 0 },
+            expect.any(Function),
+        );
+
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button/click',
             'single',
@@ -856,6 +1004,23 @@ describe('HomeAssistant extension', () => {
         );
     });
 
+    it('Should not discover sensor_click when legacy: false is set', async () => {
+        settings.set(['devices', '0x0017880104e45520'], {
+            legacy: false,
+            friendly_name: 'weather_sensor',
+            retain: false,
+        })
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+
+        const discovered = MQTT.publish.mock.calls.filter((c) => c[0].includes('0x0017880104e45520')).map((c) => c[0]);
+        expect(discovered.length).toBe(3);
+        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/action/config');
+        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/battery/config');
+        expect(discovered).toContain('homeassistant/sensor/0x0017880104e45520/linkquality/config');
+    });
+
     it('Should disable Home Assistant legacy triggers', async () => {
         settings.set(['advanced', 'homeassistant_legacy_triggers'], false);
         controller = new Controller(false);
@@ -870,16 +1035,17 @@ describe('HomeAssistant extension', () => {
         MQTT.publish.mockClear();
 
         const device = zigbeeHerdsman.devices.WXKG11LM;
+        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
 
         const discoverPayload = {
             "automation_type":"trigger",
-            "type":"click",
+            "type":"action",
             "subtype":"single",
             "payload":"single",
-            "topic":"zigbee2mqtt/button/click",
+            "topic":"zigbee2mqtt/button/action",
             "device":{
                 "identifiers":[
                     "zigbee2mqtt_0x0017880104e45520"
@@ -892,14 +1058,14 @@ describe('HomeAssistant extension', () => {
         };
 
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'homeassistant/device_automation/0x0017880104e45520/click_single/config',
-            JSON.stringify(discoverPayload),
+            'homeassistant/device_automation/0x0017880104e45520/action_single/config',
+            stringify(discoverPayload),
             { retain: true, qos: 0 },
             expect.any(Function),
         );
 
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/button/click',
+            'zigbee2mqtt/button/action',
             'single',
             { retain: false, qos: 0 },
             expect.any(Function),
@@ -907,7 +1073,7 @@ describe('HomeAssistant extension', () => {
 
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/button',
-            JSON.stringify({click: "single", linkquality: 10}),
+            stringify({action: "single", linkquality: 10}),
             { retain: false, qos: 0 },
             expect.any(Function),
         );
@@ -921,11 +1087,11 @@ describe('HomeAssistant extension', () => {
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', JSON.stringify({state: 'ON', brightness: 20}));
+        await MQTT.events.message('zigbee2mqtt/U202DST600ZB/l2/set', stringify({state: 'ON', brightness: 20}));
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', JSON.stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', JSON.stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', JSON.stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB', stringify({state_l2:"ON", brightness_l2:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l2', stringify({state:"ON", brightness:20}), {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/U202DST600ZB/l1', stringify({}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Shouldnt crash in onPublishEntityState on group publish', async () => {
@@ -935,29 +1101,92 @@ describe('HomeAssistant extension', () => {
         logger.error.mockClear();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
 
-    it('Should counter an action/click payload with an empty payload', async () => {
+    it('Should counter an action payload with an empty payload', async () => {
         controller = new Controller(false);
         await controller.start();
         await flushPromises();
         MQTT.publish.mockClear();
         const device = zigbeeHerdsman.devices.WXKG11LM;
+        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({action: 'single', linkquality: 10});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({action: '', linkquality: 10});
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
-        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/click_single/config');
-        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/click');
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('homeassistant/device_automation/0x0017880104e45520/action_single/config');
+        expect(MQTT.publish.mock.calls[3][0]).toStrictEqual('zigbee2mqtt/button/action');
+    });
+
+    it('Load Home Assistant mapping from external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const beforeCount = Object.entries((new HomeAssistant(null, null, null, null, {on: () => {}}))._getMapping()).length;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller();
+        const ha = controller.extensions.find((e) => e.constructor.name === 'HomeAssistant');
+        await controller.start();
+        await flushPromises();
+        const afterCount = Object.entries(ha._getMapping()).length;
+        expect(beforeCount + 1).toStrictEqual(afterCount);
+
+        const homeassistantSwitch = {
+            type: 'switch',
+            object_id: 'switch',
+            discovery_payload: {
+                payload_off: 'OFF',
+                payload_on: 'ON',
+                value_template: '{{ value_json.state }}',
+                command_topic: true,
+            },
+        };
+        expect(ha._getMapping()['external_converters_device']).toStrictEqual([homeassistantSwitch]);
+    });
+
+    it('Should clear outdated configs', async () => {
+        controller = new Controller(false);
+        await controller.start();
+        await flushPromises();
+
+        // Non-existing device -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability_topic: 'zigbee2mqtt/0x123/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x123/temperature/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Existing device -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/binary_sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability_topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Non-existing device of different instance -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', stringify({availability_topic: 'zigbee2mqtt_different/0x123/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+
+        // Existing device but non-existing config -> don't clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', stringify({availability_topic: 'zigbee2mqtt/0x000b57fffec6a5b2/availability'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('homeassistant/sensor/0x000b57fffec6a5b2/update_available/config', null, {qos: 0, retain: true}, expect.any(Function));
+
+        // Non-existing device but invalid payload -> clear
+        MQTT.publish.mockClear();
+        await MQTT.events.message('homeassistant/sensor/0x123/temperature/config', '1}3');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 });

test/legacy/bridgeLegacy.test.js

@@ -2,6 +2,7 @@ const data = require('../stub/data');
 const logger = require('../stub/logger');
 const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 const MQTT = require('../stub/mqtt');
+const stringify = require('json-stable-stringify');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
@@ -29,7 +30,7 @@ describe('Bridge legacy', () => {
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          JSON.stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"log_level":'info',"permit_join":false}),
+          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -56,7 +57,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -67,7 +68,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -85,12 +86,12 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
@@ -100,17 +101,17 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
@@ -187,25 +188,25 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 12, "friendly_name": "thermostat_group", "retain": false, "devices": ['TS0601_thermostat'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
     });
 
     it('Should allow rename devices', async () => {
         const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
         MQTT.publish.mockClear();
         expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color2'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(null);
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
@@ -213,7 +214,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
     });
@@ -221,12 +222,12 @@ describe('Bridge legacy', () => {
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
         expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
         expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -243,7 +244,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -265,7 +266,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -283,7 +284,7 @@ describe('Bridge legacy', () => {
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -298,7 +299,7 @@ describe('Bridge legacy', () => {
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -318,10 +319,24 @@ describe('Bridge legacy', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
         await flushPromises();
         expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
         expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            stringify({type: 'group_removed', message: 'group_1'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -356,7 +371,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_removed', message: 'bulb_color'}),
+            stringify({type: 'device_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -378,7 +393,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            stringify({type: 'device_force_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -399,7 +414,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_banned', message: 'bulb_color'}),
+            stringify({type: 'device_banned', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );

test/networkMap.test.js

@@ -1,6 +1,7 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify');
 
 const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
 
@@ -26,6 +27,7 @@ describe('Networkmap', () => {
         data.writeDefaultConfiguration();
         settings._reRead();
         data.writeEmptyState();
+        settings.set(['experimental', 'new_api'], true);
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
@@ -75,7 +77,7 @@ describe('Networkmap', () => {
         unsupported_router.routingTable = () => {throw new Error('failed')};
     }
 
-    it('Output raw networkmap', async () => {
+    it('Output raw networkmap legacy api', async () => {
         mock();
         MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
         await flushPromises();
@@ -126,11 +128,7 @@ describe('Networkmap', () => {
         expect(JSON.parse(call[1])).toStrictEqual(expected);
     });
 
-    it('Output graphviz networkmap', async () => {
-        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
-        const getHours = Date.prototype.getHours;
-        Date.prototype.getTimezoneOffset = () => -60;
-        Date.prototype.getHours = () => 1;
+    it('Output graphviz networkmap legacy api', async () => {
         mock();
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = null;
@@ -146,16 +144,16 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
               "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
               "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
             }`;
 
@@ -165,7 +163,149 @@ describe('Networkmap', () => {
         for (let i = 0; i < expectedLines.length; i++) {
             expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
         }
-        Date.prototype.getTimezoneOffset = getTimezoneOffset;
-        Date.prototype.getHours = getHours;
+    });
+
+    it('Output plantuml networkmap legacy api', async () => {
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'plantuml');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/plantuml');
+
+        const expected = `' paste into: https://www.planttext.com/
+
+        @startuml
+        card 0x0017880104e45525 [
+        0x0017880104e45525
+        ---
+        0x0017880104e45525 (6536) failed: lqi,routingTable
+        ---
+        Boef notSupportedModelID
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b2 [
+        bulb
+        ---
+        0x000b57fffec6a5b2 (40369)
+        ---
+        IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b3 [
+        bulb_color
+        ---
+        0x000b57fffec6a5b3 (40399)
+        ---
+        Philips Hue Go (7146060PH)
+        ---
+        unknown
+        ]
+
+        card 0x0017880104e45521 [
+        button_double_key
+        ---
+        0x0017880104e45521 (6538)
+        ---
+        Xiaomi Aqara double key wireless wall switch (WXKG02LM)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x0017880104e45559 [
+        cc2530_router
+        ---
+        0x0017880104e45559 (6540)
+        ---
+        Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x00124b00120144ae [
+        Coordinator
+        ---
+        0x00124b00120144ae (0)
+        ---
+        0 seconds ago
+        ]
+
+        0x000b57fffec6a5b3 --> 0x00124b00120144ae: 120
+        0x000b57fffec6a5b2 --> 0x00124b00120144ae: 92
+        0x000b57fffec6a5b3 --> 0x000b57fffec6a5b2: 110
+        0x0017880104e45559 --> 0x000b57fffec6a5b2: 100
+        0x0017880104e45521 --> 0x0017880104e45559: 130
+
+        @enduml`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+    });
+
+    it('Should output raw networkmap', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', stringify({type: 'raw', routes: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
+
+        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
+
+        const expected = {
+            status: 'ok',
+            data: {
+                type: 'raw',
+                routes: true,
+                value: {
+                    "nodes":[
+                        {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
+                        {"lastSeen": null,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
+                        {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
+                        {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
+                        {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
+                    ],
+                    "links":[
+                        {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
+                        {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
+                        {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
+                        {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
+                        {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
+                    ]
+                }
+            }
+        };
+
+        const actual = JSON.parse(call[1]);
+        actual.data.value = JSON.parse(actual.data.value);
+        expect(actual).toStrictEqual(expected);
+    });
+
+    it('Should throw error when rquesting invalid type', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', 'not_existing');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/networkmap',
+            stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/otaUpdate.test.js

@@ -6,6 +6,7 @@ const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const flushPromises = () => new Promise(setImmediate);
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify');
 
 describe('OTA update', () => {
     let controller;
@@ -18,6 +19,7 @@ describe('OTA update', () => {
     beforeEach(async () => {
         data.writeDefaultConfiguration();
         settings._reRead();
+        settings.set(['experimental', 'new_api'], true);
         data.writeEmptyState();
         controller = new Controller();
         await controller.start();
@@ -28,6 +30,8 @@ describe('OTA update', () => {
     it('Should subscribe to topics', async () => {
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
         expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/ota_update/update');
     });
 
     it('Should OTA update a device', async () => {
@@ -41,14 +45,13 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
         logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             onUpdate(0, null);
-            onUpdate(10, 3600);
+            onUpdate(10, 3600.2123);
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', 'bulb');
         await flushPromises();
         expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
@@ -56,11 +59,30 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
-        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
         expect(device.softwareBuildID).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"updating","progress":10,"remaining":3600}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle when OTA update fails', async () => {
@@ -69,17 +91,23 @@ describe('OTA update', () => {
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        logger.error.mockClear();
         device.save.mockClear();
         mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
             throw new Error('Update failed');
         });
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', stringify({id: "bulb"}));
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":true,"update":{"state":"available"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should be able to check if OTA update is available', async () => {
@@ -87,41 +115,65 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
 
-        logger.info.mockClear();
+        MQTT.publish.mockClear();
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.error.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
 
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
-        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail when device does not exist', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "not_existing_deviceooo");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "ZNLDP12LM");
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "ZNLDP12LM"},"status":"error","error": `Device 'ZNLDP12LM' does not support OTA updates`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should refuse to check/update when already in progress', async () => {
@@ -130,36 +182,38 @@ describe('OTA update', () => {
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
 
-        logger.info.mockClear();
         mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
             return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
         });
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
         await flushPromises();
         expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
         jest.runAllTimers();
         await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
-    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+    it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         let count = 0;
-        endpoint.read.mockImplementation(() => {
-            if (count === 1) throw new Error('Failed!')
-            count++;
-            return {swBuildId: 1, dateCode: '2019010'}
-        });
+        endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
 
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
-        logger.info.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', "bulb");
         await flushPromises();
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should check for update when device requests it', async () => {
@@ -189,7 +243,12 @@ describe('OTA update', () => {
         mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`)
+        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":true,"update":{"state":"available"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
     });
 
     it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
@@ -212,4 +271,116 @@ describe('OTA update', () => {
         expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
         expect(logger.error).toHaveBeenCalledTimes(0);
     });
+
+    it('Legacy api: Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Legacy api: Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Legacy api: Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Legacy api: Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Legacy api: Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Legacy api: Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
 });

test/receive.test.js

@@ -1,5 +1,6 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
@@ -28,9 +29,7 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single', click: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should handle a zigbee message which uses ep (left)', async () => {
@@ -41,7 +40,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left', linkquality: 10});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -53,7 +52,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right', linkquality: 10});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -69,6 +68,35 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
+    it('Should allow to invert cover', async () => {
+        const device = zigbeeHerdsman.devices.J1;
+
+        // Non-inverted (open = 100, close = 0)
+        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+
+        // Inverted
+        MQTT.publish.mockClear();
+        settings.set(['devices', device.ieeeAddr, 'invert_cover'], true);
+        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80, linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to disable the legacy integration', async () => {
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
+    });
+
     it('Should debounce messages', async () => {
         jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
@@ -148,7 +176,7 @@ describe('Receive', () => {
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
         jest.runAllTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
@@ -336,13 +364,18 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    it('Should not handle messages forwarded Xiaomi messages', async () => {
+    it('Should handle forwarded Xiaomi messages', async () => {
         const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const data = {onOff: 1};
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 599};
+        const payload = {data: {measuredValue: -85}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 6539};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/weather_sensor',
+            stringify({temperature: -0.85, linkquality: 10}),
+            {"qos": 1, "retain": false},
+            expect.any(Function),
+        )
     });
 
     it('Should handle messages from Xiaomi router devices', async () => {
@@ -366,23 +399,27 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from unsupported devices', async () => {
+    it('Should not handle messages from unsupported devices and link to docs', async () => {
         const device = zigbeeHerdsman.devices.unsupported;
         const data = {onOff: 1};
+        logger.warn.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+    it('Should not handle messages from still interviewing devices with unknown definition', async () => {
         const device = zigbeeHerdsman.devices.interviewing;
         const data = {onOff: 1};
+        logger.debug.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, definition is undefined and still interviewing`);
     });
 
     it('Should handle a command', async () => {
@@ -393,7 +430,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop', linkquality: 10});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -411,10 +448,10 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(JSON.parse(MQTT.publish.mock.calls[1][1]).elapsed).toBe(50);
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         Date.now = oldNow;

test/report.test.js

@@ -24,6 +24,9 @@ describe('Report', () => {
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(1);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(0);
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
         expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
@@ -38,12 +41,32 @@ describe('Report', () => {
         }
     }
 
+    function expectOnOffBrightnessColorReportDisabled(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
     mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
             endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
         }
     }
 
@@ -52,6 +75,11 @@ describe('Report', () => {
         settings._reRead();
         data.writeEmptyState();
         settings.set(['advanced', 'report'], true);
+        for (const device of Object.values(zigbeeHerdsman.devices)) {
+            mockClear(device);
+            delete device.meta.reporting;
+        }
+
         controller = new Controller();
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
@@ -64,6 +92,31 @@ describe('Report', () => {
         expectOnOffBrightnessColorReport(endpoint, true);
     });
 
+    it('Should not configure reporting on startup when disabled', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        delete device.meta.report;
+        settings.set(['advanced', 'report'], false);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should disable reporting on startup when enabled earlier', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.meta.reporting = 1;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['advanced', 'report'], false);
+        mockClear(device);
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expectOnOffBrightnessColorReportDisabled(endpoint, true);
+    });
+
     it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
@@ -108,6 +161,20 @@ describe('Report', () => {
         expect(device.meta.reporting).toBeUndefined();
     });
 
+    it('Should not configure reporting when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        device.interviewing = true;
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
     it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);

test/settings.test.js

@@ -9,6 +9,13 @@ const devicesFile = data.joinPath('devices.yaml');
 const groupsFile = data.joinPath('groups.yaml');
 const secretFile = data.joinPath('secret.yaml');
 const yaml = require('js-yaml');
+const objectAssignDeep = require(`object-assign-deep`);
+
+const minimalConfig = {
+    permit_join: true,
+    homeassistant: true,
+    mqtt: {base_topic: 'zigbee2mqtt', server: 'localhost'},
+};
 
 describe('Settings', () => {
     const write = (file, json, reread=true) => {
@@ -21,11 +28,19 @@ describe('Settings', () => {
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
+    const clearEnvironmentVariables = () => {
+        Object.keys(process.env).forEach((key) => { 
+            if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
+                delete process.env[key];
+            }
+        });
+    }
 
     beforeEach(() => {
         remove(configurationFile);
         remove(devicesFile);
         remove(groupsFile);
+        clearEnvironmentVariables();
     });
 
     it('Should return default settings', () => {
@@ -47,6 +62,29 @@ describe('Settings', () => {
         expect(s).toStrictEqual(expected);
     });
 
+    it('Should apply environment variables', () => {
+        process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
+        process.env['ZIGBEE2MQTT_CONFIG_EXPERIMENTAL_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_AVAILABILITY_BLOCKLIST'] = '["0x43597f0dac781b1e", "x223b0aef2ae8d1b0"]';
+        process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+
+        write(configurationFile, {});
+        const s = settings.get();
+        const expected = objectAssignDeep.noMutate({}, settings._getDefaults());
+        expected.devices = {};
+        expected.groups = {};
+        expected.serial.disable_led = true;
+        expected.advanced.soft_reset_timeout = 1;
+        expected.experimental.output = 'csvtest';
+        expected.advanced.availability_blocklist = ['0x43597f0dac781b1e', 'x223b0aef2ae8d1b0'];
+        expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
+        expected.mqtt.base_topic = 'testtopic';
+
+        expect(s).toStrictEqual(expected);
+    });
+
     it('Should add devices', () => {
         write(configurationFile, {});
         settings.addDevice('0x12345678');
@@ -359,6 +397,14 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Should throw error when changing entity options of non-existing device', () => {
+        write(configurationFile, {});
+
+        expect(() => {
+            settings.changeEntityOptions('not_existing_123', {});
+        }).toThrow(new Error("Device or group 'not_existing_123' does not exist"));
+    });
+
     it('Should not add duplicate groups', () => {
         write(configurationFile, {});
 
@@ -497,40 +543,63 @@ describe('Settings', () => {
 
     it('Should not allow any string values for network_key', () => {
         write(configurationFile, {
+            ...minimalConfig,
             advanced: {network_key: 'NOT_GENERATE'},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+        const error = `advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Should allow retention configuration with MQTT v5', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            mqtt: {base_topic: 'zigbee2mqtt', server: 'localhost', version: 5},
+            devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
+        });
+
+        settings._reRead();
+        expect(settings.validate()).toEqual([]);
     });
 
 
     it('Should not allow retention configuration without MQTT v5', () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError('MQTT retention requires protocol version 5');
+        const error = 'MQTT retention requires protocol version 5';
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
-    it('Should not allow non-existing entities in availability blacklist', () => {
+    it('Should not allow non-existing entities in availability_blocklist', () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
-            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+            advanced: {availability_blocklist: ['0x0017880104e45519', 'non_existing']},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+        const error = `Non-existing entity 'non_existing' specified in 'availability_blocklist'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Validate should if settings does not conform to scheme', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            advanced: null,
+        });
+
+        settings._reRead();
+
+        const error = `advanced should be object`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should ban devices', () => {
@@ -541,80 +610,78 @@ describe('Settings', () => {
         expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
     });
 
+    it('Should add devices to blocklist', () => {
+        write(configurationFile, {});
+        settings.blockDevice('0x123');
+        expect(settings.get().blocklist).toStrictEqual(['0x123']);
+        settings.blockDevice('0x1234');
+        expect(settings.get().blocklist).toStrictEqual(['0x123', '0x1234']);
+    });
+
     it('Should throw error when yaml file is invalid', () => {
         fs.writeFileSync(configurationFile, `
              good: 9
              \t wrong
         `)
 
-        let error;
-        try {
-            settings._reRead();
-        } catch (e) {
-            error = e;
-        }
-
-        expect(error.message).toContain(`Your YAML file '${configurationFile}' is invalid`);
+        settings._clear();
+        const error = `Your YAML file: '${configurationFile}' is invalid (use https://jsonformatter.org/yaml-validator to find and fix the issue)`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should throw error when yaml file does not exist', () => {
-        let error;
-        try {
-            settings._reRead();
-        } catch (e) {
-            error = e;
-        }
-
-        expect(error.message).toContain(`no such file or directory, open`);
+        settings._clear();
+        const error = `ENOENT: no such file or directory, open '${configurationFile}'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname', retain: false}},
             groups: {'1': {friendly_name: 'myname', retain: false}},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Duplicate friendly_name 'myname' found`);
+        const error = `Duplicate friendly_name 'myname' found`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
+        const error = `Friendly name cannot end with a "/DIGIT" ('myname/123')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+        const error = `MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
         });
 
         settings._reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
+        const error = `Following friendly_name are not allowed: '${utils.getEndpointNames()}'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {

test/stub/data.js

@@ -2,9 +2,11 @@ const tmp = require('tmp');
 const yaml = require('../../lib/util/yaml');
 const path = require('path');
 const fs = require('fs');
+const stringify = require('json-stable-stringify');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
+const stateFile = path.join(mockDir, 'state.json');
 
 function writeDefaultConfiguration() {
     const config = {
@@ -108,7 +110,7 @@ function writeDefaultConfiguration() {
             },
             '0x0017880104e45551': {
                 retain: false,
-                friendly_name: 'smart_vent'
+                friendly_name: 'smart vent'
             },
             '0x0017880104e45552': {
                 retain: false,
@@ -153,7 +155,13 @@ function writeDefaultConfiguration() {
             },
             '0x0017880104e44559': {
                 friendly_name: '3157100_thermostat',
-            }
+            },
+            '0x0017880104a44559': {
+                friendly_name: 'J1_cover',
+            },
+            '0x0017882104a44559': {
+                friendly_name: 'TS0601_thermostat',
+            },
         },
         groups: {
             '1': {
@@ -174,14 +182,30 @@ function writeDefaultConfiguration() {
                 retain: false,
                 devices: ['bulb_2']
             },
-        }
+            '12': {
+                friendly_name: 'thermostat_group',
+                retain: false,
+                devices: ['TS0601_thermostat'],
+            }
+        },
+        external_converters: [],
     };
 
     yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
 }
 
 function writeEmptyState() {
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
+    fs.writeFileSync(stateFile, stringify({}));
+}
+
+function removeState() {
+    if (stateExists()) {
+        fs.unlinkSync(stateFile);
+    }
+}
+
+function stateExists() {
+    return fs.existsSync(stateFile);
 }
 
 function writeDefaultState() {
@@ -197,11 +221,7 @@ function writeDefaultState() {
         },
     }
 
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
-}
-
-function removeState() {
-    fs.unlinkSync(path.join(mockDir, 'state.json'))
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(state));
 }
 
 jest.mock('../../lib/util/data', () => ({
@@ -219,4 +239,5 @@ module.exports = {
     writeDefaultState,
     removeState,
     writeEmptyState,
+    stateExists,
 };

test/stub/logger.js

@@ -1,14 +1,26 @@
 let level = 'info';
 
+const transports = [];
+
+let transportsEnabled = false;
+const callTransports = (level, message) => {
+    if (transportsEnabled) {
+        for (const transport of transports) {
+            transport.log({level, message}, () => {});
+        }
+    }
+}
+
 const mock = {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn(),
-    debug: jest.fn(),
-    trace: jest.fn(),
+    info: jest.fn().mockImplementation((msg) => callTransports('info', msg)),
+    warn: jest.fn().mockImplementation((msg) => callTransports('warn', msg)),
+    error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),
+    debug: jest.fn().mockImplementation((msg) => callTransports('debug', msg)),
     cleanup: jest.fn(),
+    add: (transport) => transports.push(transport),
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,
+    setTransportsEnabled: (value) => {transportsEnabled = value},
 };
 
 jest.mock('../../lib/util/logger', () => (mock));

test/stub/mqtt.js

@@ -4,6 +4,7 @@ const mock = {
     publish: jest.fn().mockImplementation((topic, payload, options, cb) => cb()),
     end: jest.fn(),
     subscribe: jest.fn(),
+    reconnecting: false,
     on: (type, handler) => {
         if (type === 'connect') {
             handler();
@@ -20,5 +21,5 @@ jest.mock('mqtt', () => {
 });
 
 module.exports = {
-    events, ...mock, connect: mockConnect,
+    events, ...mock, connect: mockConnect, mock,
 };

test/stub/zigbeeHerdsman.js

@@ -1,6 +1,11 @@
 const events = {};
 const assert = require('assert');
 
+function getKeyByValue(object, value, fallback) {
+    const key = Object.keys(object).find((k) => object[k] === value);
+    return key != null ? key : fallback;
+}
+
 class Group {
     constructor(groupID, members) {
         this.groupID = groupID;
@@ -8,6 +13,7 @@ class Group {
         this.meta = {};
         this.members = members;
         this.removeFromDatabase = jest.fn();
+        this.removeFromNetwork = jest.fn();
         this.hasMember = (endpoint) => this.members.includes(endpoint);
     }
 }
@@ -37,6 +43,14 @@ class Endpoint {
         this.unbind = jest.fn();
         this.configureReporting = jest.fn();
         this.binds = binds;
+        this.getInputClusters = () => inputClusters.map((c) => {
+            return {ID: c, name: getKeyByValue(clusters, c)};
+        }).filter((c) => c.name);
+
+        this.getOutputClusters = () => outputClusters.map((c) => {
+            return {ID: c, name: getKeyByValue(clusters, c)};
+        }).filter((c) => c.name);
+
         this.supportsInputCluster = (cluster) => {
             assert(clusters[cluster] !== undefined, `Undefined '${cluster}'`);
             return this.inputClusters.includes(clusters[cluster]);
@@ -47,20 +61,22 @@ class Endpoint {
             return this.outputClusters.includes(clusters[cluster]);
         }
 
-        this.addToGroup = (group) => {
+        this.addToGroup = jest.fn();
+        this.addToGroup.mockImplementation((group) => {
             if (!group.members.includes(this)) group.members.push(this);
-        }
+        })
 
         this.getDevice = () => {
             return Object.values(devices).find(d => d.ieeeAddr === deviceIeeeAddress);
         }
 
-        this.removeFromGroup = (group) => {
+        this.removeFromGroup = jest.fn();
+        this.removeFromGroup.mockImplementation((group) => {
             const index = group.members.indexOf(this);
             if (index != -1) {
                 group.members.splice(index, 1);
             }
-        }
+        });
 
         this.removeFromAllGroups = () => {
             Object.values(groups).forEach((g) => this.removeFromGroup(g))
@@ -103,6 +119,7 @@ const returnDevices = [];
 const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
 const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
 const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
+const TS0601_thermostat =  new Device('EndDevice', '0x0017882104a44559', 6544,4151, [new Endpoint(1, [], [], '0x0017882104a44559')], true, "Mains (single phase)", 'kud7u2l');
 
 const devices = {
     'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
@@ -110,7 +127,7 @@ const devices = {
     'bulb_color': bulb_color,
     'bulb_2': bulb_2,
     'bulb_color_2': bulb_color_2,
-    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0]}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
+    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0], cluster: {ID: 8, name: 'genLevelCtrl'}}, {target: new Group(1, []), cluster: {ID: 6, name: 'genOnOff'}}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
     'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
     'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
@@ -125,7 +142,7 @@ const devices = {
     'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
     'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
     'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
-    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
+    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45545')], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
     'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
     'HAMPTON99432':  new Device('Router', '0x0017880104e45548', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'HDC52EastwindFan'),
     'HS2WD': new Device('Router', '0x0017880104e45549', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'WarningDevice'),
@@ -146,13 +163,16 @@ const devices = {
     'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
     'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
     '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
+    'J1': new Device('Router', '0x0017880104a44559', 6543,4151, [new Endpoint(1, [], [], '0x0017880104a44559')], true, "Mains (single phase)", 'J1 (5502)'),
+    'TS0601_thermostat': TS0601_thermostat,
 }
 
 const groups = {
     'group_1': new Group(1, []),
     'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
     'group/with/slashes': new Group(99, []),
-    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]])
+    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]]),
+    'thermostat_group': new Group(12, [TS0601_thermostat.endpoints[0]])
 }
 
 const mock = {
@@ -176,6 +196,9 @@ const mock = {
     getDeviceByIeeeAddr: jest.fn().mockImplementation((ieeeAddr) => {
         return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.ieeeAddr === ieeeAddr);
     }),
+    getDeviceByNetworkAddress: jest.fn().mockImplementation((networkAddress) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.networkAddress)).find((d) => d.networkAddress === networkAddress);
+    }),
     getGroups: jest.fn().mockImplementation((query) => {
         return Object.values(groups);
     }),

update.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 
-echo "Stopping zigbee2mqtt..."
+echo "Stopping Zigbee2MQTT..."
 sudo systemctl stop zigbee2mqtt
 
 echo "Creating backup of configuration..."
@@ -17,7 +17,7 @@ echo "Restore configuration..."
 cp -R data-backup/* data
 rm -rf data-backup
 
-echo "Starting zigbee2mqtt..."
+echo "Starting Zigbee2MQTT..."
 sudo systemctl start zigbee2mqtt
 
 echo "Done!"