1.14.0

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.dockerignore

@@ -1,7 +1,15 @@
-node_modules
-.travis*
+.eslintignore
+.eslintrc.json
 .git
+.github
 .gitignore
-*.png
-firmware
-docker/Dockerfile*
+.npmrc
+.travis*
+.travis.yml
+README.md
+docker/Dockerfile*
+images
+node_modules
+scripts
+test
+update.sh

.eslintignore

@@ -1 +1,2 @@
-node_modules/*
+node_modules/*
+test/

.github/ISSUE_TEMPLATE.md

@@ -1,23 +1,38 @@
 <!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2mqtt version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
 Did you read the FAQ?
-https://www.zigbee2mqtt.io/information/FAQ.html
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2mqtt fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
 
 Unsupported device?
-https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
 
-A question?
-Read the documentation: https://www.zigbee2mqtt.io/
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
 
-Bug report:
-- First make sure you are on the latest version of zigbee2mqtt.
-- Make sure you are running the latest CC253X firmware from https://github.com/koenkk/z-stack-firmware.
-- Provide a clear description of the problem.
+Bug report?
 - If applicable, provide steps how to reproduce the problem.
-- Provide a log with `log_level` set to `debug`, see https://www.zigbee2mqtt.io/configuration/configuration.html
-- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
 -->
 
 <!--Start your bug report below this line-->
+
 # Bug Report
 ## What happened
 
@@ -26,5 +41,6 @@ Bug report:
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-zigbee2mqtt version:
-CC253X firmware version:
+Zigbee2mqtt version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.github/branch-switcher.yml

@@ -0,0 +1 @@
+preferredBranch: dev

.gitignore

@@ -65,4 +65,9 @@ data/state.json
 data/log
 data-backup/
 data/coordinator_backup.json
-data/.storage
+data/.storage
+data/database.db.backup
+data/extension
+
+# MacOS indexing file
+.DS_Store

.npmignore

@@ -0,0 +1,25 @@
+#tests
+test
+coverage
+
+#build tools
+.travis.yml
+.jenkins.yml
+.codeclimate.yml
+.github
+azure-pipelines.yaml
+
+#linters
+.jscsrc
+.jshintrc
+.eslintrc*
+.dockerignore
+.eslintignore
+
+#editor settings
+.vscode
+
+#src
+docs
+docker
+images

.travis.yml

@@ -1,33 +0,0 @@
-language: node_js
-
-sudo: required
-
-services:
-  - docker
-
-node_js:
-  - "10"
-
-before_install:
-  - "echo '{\"experimental\": true}' | sudo tee /etc/docker/daemon.json"
-  - sudo apt update -y
-  - sudo apt install --only-upgrade docker-ce -y
-  - sudo service docker restart
-
-install:
-  - npm install
-
-before_script:
-  - npm test
-  - npm run eslint
-
-script:
-  - ".travis/docker.sh"
-  - ".travis/trigger-downstream.sh"
-
-env:
-  global:
-    - DOCKER_USERNAME=koenkk
-    - secure: q5W9+mHz13PslLkTxuYNXivOau+N5OyB53sqgb+krOCPlbN9+gvi2Av9SribVd5jp2yx2O9rV/mr8eS0M2ykvBJ3UsIsCXiFL1dEPmsz1wlFAnW3kbD2EGhGFEV0dWRflSlA+5iQtcMPvbYu2PAOChUUoY7NNlhcz2EC+XXUapvvvaDwA1oSdtX+m61qyREshZb8B57DyXhziy6i/xjhFa84C0mR/iGFRWpZgcxmrNFH+/XDhSIpdq2KshhDWhjyag3ukAsTxTpqzLh1rFRlczjJEn0CcHG4WHsWHUFn9KK2lvT/YCDP8w0URnR1bCr8nyxGZJ2DLW6AzxZtJfWjvbeZYjO2R3WeHpzp6Jw6Qz5pG3gmu2FgeFpZ3FTEGtJ/Ev0/pjkY2Msx9mItzUJ7D8hY2noQZMPASar6oXEDYlkomx7gHHla0MPv3PKSJg5LX6Ybl137GIa6MLYYyz6SEKfyRUQhy3Itqrb/ID/ZP0UuLgyyDfShb8A6XOnsVqwZAsOPtU2UPnD1vsVXhjkpflK9+7QjL5m9MSqfAhg6qtwNsswl1KSiyhmdldGhZW5OfUasDxZKHyx7bsS1BhNnLi/iy4ifDh6KSr64aUW4iM0X99gkU+662Hgf73QpIG+oP41r30iEN0unTcRrFQDIbTSSapbA95bwUCBF1CgQC20=
-    - secure: em3lBVXvZ3kOVI4ipVlh93NIU6guLvyzHMmsLDuxhc92VFURqlYu3iXTTNEnknwyE31mziAkVFnmj+3OENd/VfzP01d4KKaYAQrbn2AnOMvcIIExLfaFU5QqbW1RMwRBIifhvrmboKt07pFsUy3Rghs80SwNSFDfLFoMhFMz+NdIZfJGW8LJy+gUPmz10YGzqh/chreowqOemgBBoswCGwIh1xB1KNejew3bQllnwQYGXcGuTDwDT4Wz+ObxlFt7YR/4n4oJY8jQYRPolM9qHQAGyn/Cn0TL3c8JqiBpVc1hm2OhYSDqvK0xf5IKsD+kIfFJPGa7nF2cwmhKdUNYR47oue3KB5tdrtDhwqUooTglvBOSXUIbthUR9Xu40ziyd5tHy0ixaLnDRGuKIIvEnXlcE6CgApoe3y2kHYa5dY+eUKxWwvH7b5hG+u82dYvw3VYpCHAnDQVgUzz4F63qZ2WZhcH0W3PF3/ytVAHyMu2vAMSrSXGi5xgM6el7Gp7SFVTFXBurbOaoJ3ZwwSEDP5d8mDyk1d8thmkQFL9P6iWtNnXOzjv4GIliEDWQg76BX6MLD3n3fnP0aq+RVmv+ev2Lp4xZZDfgGPek7g35IsaiohH6YvHloxE+ETQ0Ttb1lCQos9fj7LlI+ud6ExjwyRuEnzCOzr/qC/3Leg85Wc8=
-    - secure: GbJVFFY4nmQovp8i2VoQ1AOV8h9QoAcleBV9Mg5TItA8U2Y+6t9wx0iWbaclxOrzI727XZOk5HquMTKKTI430lT6OjU01+YJqwyq+k650YiX5PImsgJXGhFzBKeUcB3A53kKijfr9aEV00YiXRsKoKAn+Bl09m/9O+melBbTwO/YAQyt6EoUt21Yk0BoqpW2CTMPAgsU5tm6Ra3bbJ+IaUYVMYBtuHqtGiKg8jgTHqRhwKejDyV7+1dDxX8q51bK2Kb6PujbI1SS5wsJEpkU/eoPs7MuprmlYarYtBt1fDXSK1EOjcfpyEFRlrNLewIKiFEo3dkTOEtW2a4t3tcAiXOj8Hcvp+6RsjTfnjYmZweHVh/JKDhujjHBMviyTjwp4RnPaQSFou0iVIB5+Mo34hUhFDXc30SBzaux+lyR3rGoEu5pubZNjG5CkKIpw3IyduNKtIrlbCnfg+WDU3jrXmk0fzmwyriyXhx9BAzM+hyFiElZ+3xoXAlmDlwvaRZyJuQQNslwGPbZyX9EByIUkECjYPf92QPyD0mjbn8C32IphgQ515Up+15g2meDniugzowLJkDe3W86M5fB1kPQiDq5oT4M6TgAPRElLN6/ibyfOzDAtOdZgUiQvFnUQb1s9LunZqrc57AKzuzl2xG9Ok/cUkZFSOjrFr40jXgS2Lw=

.travis/docker.sh

@@ -1,68 +0,0 @@
-#!/bin/bash -e
-
-login() {
-  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-}
-
-tag_push() {
-  docker tag $DOCKER_USERNAME/zigbee2mqtt:$1 $DOCKER_USERNAME/zigbee2mqtt:$2
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$2
-}
-
-build_and_push() {
-  docker build --platform=$3 --build-arg COMMIT=$(git rev-parse --short HEAD) -t $DOCKER_USERNAME/zigbee2mqtt:$1 -f $2 .
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$1
-}
-
-push_manifest() {
-  export DOCKER_CLI_EXPERIMENTAL=enabled  
-
-  docker manifest create $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1
-
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 --os linux --arch arm --variant v6
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1 --os linux --arch arm64 --variant v8
-
-  docker manifest inspect $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker manifest push -p $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker run --rm mplatform/mquery $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  export DOCKER_CLI_EXPERIMENTAL=disabled
-}
-
-# Only update Docker images for:
-# - dev branch
-# - version release
-if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ ! -z ${TRAVIS_TAG+x} ] && [ ! "$TRAVIS_TAG" = "" ]
-then
-  echo "Updating docker images for master branch!"
-  login
-
-  # Push versioned images
-  build_and_push "$TRAVIS_TAG" docker/Dockerfile.amd64 amd64
-  build_and_push "$TRAVIS_TAG-arm32v6" docker/Dockerfile.arm32v6 arm
-  build_and_push "$TRAVIS_TAG-arm64v8" docker/Dockerfile.arm64v8 arm64
-
-  # Push latest images.
-  tag_push "$TRAVIS_TAG-arm32v6" "arm32v6"
-  tag_push "$TRAVIS_TAG-arm64v8" "arm64v8"
-  tag_push "$TRAVIS_TAG" latest
-
-  # Push manifest
-  push_manifest ""
-
-elif [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Updating docker images for dev branch!"
-  login
-  build_and_push latest-dev docker/Dockerfile.amd64 amd64
-  build_and_push arm32v6-dev docker/Dockerfile.arm32v6 arm
-  build_and_push arm64v8-dev docker/Dockerfile.arm64v8 arm64
-
-  # Push manifest
-  push_manifest "-dev"
-
-else
-  echo "Not updating docker images, triggered by pull request or not on master/dev branch"
-fi

.travis/trigger-downstream.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-function travis_trigger() {
-     local org=$1 && shift
-     local repo=$1 && shift
-     local branch=${1:-master} && shift
-
-     body="{
-             \"request\": {
-               \"branch\": \"${branch}\"
-              }
-           }"
-
-     curl -s -X POST \
-          -H "Content-Type: application/json" \
-          -H "Accept: application/json" \
-          -H "Travis-API-Version: 3" \
-          -H "Authorization: token $TRAVIS_TOKEN" \
-          -d "$body" \
-          "https://api.travis-ci.org/repo/${org}%2F${repo}/requests"
-}
-
-# Only trigger downstream if on dev branch and not pull request
-if [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Triggering build of downstream projects!"
-  travis_trigger "danielwelch" "hassio-zigbee2mqtt" "master"
-else
-  echo "Not triggering build of downstream projects, triggered by pull request or not on dev branch"
-fi

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "*.yaml": "home-assistant"
+    }
+}

README.md

@@ -5,8 +5,8 @@
     <br>
     <br>
     <div style="display: flex;">
-        <a href="https://travis-ci.org/Koenkk/zigbee2mqtt/">
-            <img src="https://img.shields.io/travis/Koenkk/zigbee2mqtt.svg">
+        <a href="https://dev.azure.com/koenkk/zigbee/_build?definitionId=5">
+            <img src="https://dev.azure.com/koenkk/zigbee/_apis/build/status/Koenkk.zigbee2mqtt?branchName=dev">
         </a>
         <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
             <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
@@ -23,6 +23,15 @@
         <a href="http://zigbee2mqtt.discourse.group/">
             <img src="https://img.shields.io/discourse/https/zigbee2mqtt.discourse.group/status.svg">
         </a>
+        <a>
+            <img src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg">
+        </a>
+        <a href="https://www.codacy.com/manual/Koenkk/zigbee2mqtt?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=Koenkk/zigbee2mqtt&amp;utm_campaign=Badge_Grade">
+            <img src="https://api.codacy.com/project/badge/Grade/24f1e0fe39f04daa810e8a1416693d3f">
+        </a>
+        <a href="https://www.npmjs.com/package/zigbee2mqtt">
+            <img src="https://img.shields.io/npm/v/zigbee2mqtt">
+        </a>
     </div>
     <h1>Zigbee2mqtt  🌉 🐝</h1>
     <p>
@@ -57,6 +66,9 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ## Architecture
 ![Architecture](images/architecture.png)
 
+### Internal Architecture
+Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 

azure-pipelines.yaml

@@ -0,0 +1,88 @@
+trigger:
+  tags:
+    include:
+    - '*'
+  branches:
+    include:
+    - '*'
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+steps:
+
+# Prepare
+- task: NodeTool@0
+  inputs:
+    versionSpec: '12.x'
+  displayName: 'Install Node.js'
+- bash: npm ci
+  displayName: 'Install dependencies'
+
+# Verify
+- bash: npm run test-with-coverage
+  displayName: 'Test'
+- bash: npm run eslint
+  displayName: 'Lint'
+
+# NPM
+- task: Npm@1
+  inputs:
+    command: publish
+    publishEndpoint: npm
+  displayName: 'Publish to npm'
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
+
+# Docker
+- task: DockerInstaller@0
+  displayName: Docker install
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  inputs:
+    dockerVersion: 19.03.5
+    releaseType: stable
+- task: Docker@2
+  displayName: Docker registry login
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  inputs:
+    command: login
+    containerRegistry: docker_registry
+- bash: |
+    sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
+    sudo chmod a+x /usr/local/bin/buildx
+    docker run --rm --privileged hypriot/qemu-register:v2.7.0
+    buildx create --use
+    buildx ls
+  displayName: 'Docker setup'
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+- bash: |
+    buildx build \
+      --build-arg COMMIT=$(git rev-parse --short HEAD) \
+      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+      -f docker/Dockerfile \
+      --push \
+      -t koenkk/zigbee2mqtt:latest-dev \
+      .
+  displayName: 'Docker build dev'
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))
+- bash: |
+    TAG="$(git describe --tags)"
+    buildx build \
+      --build-arg COMMIT=$(git rev-parse --short HEAD) \
+      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+      -f docker/Dockerfile \
+      --push \
+      -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
+      .
+  displayName: 'Docker build release'
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
+
+# Trigger Home Assistant Zigbee2mqtt addon build
+- bash: |
+    curl -s \
+        --request POST \
+        --header "Content-Type: application/json" \
+        --data '{"definition": {"id": 1}}' \
+        -u "$(HASSIO_TRIGGER_SECRET)" \
+        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+  displayName: 'Trigger Hass.io build'
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -0,0 +1,30 @@
+FROM node:12-alpine as base
+
+WORKDIR /app
+RUN apk add --no-cache tzdata eudev
+
+COPY package.json .
+
+# Dependencies
+FROM base as dependencies
+
+COPY npm-shrinkwrap.json .
+
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
+    npm ci --production && \
+    apk del .buildtools
+
+# Release
+FROM base as release
+
+COPY --from=dependencies /app/node_modules ./node_modules
+COPY lib ./lib
+COPY index.js docker/run.sh data/configuration.yaml ./
+
+RUN chmod +x /app/run.sh
+RUN mkdir /app/data
+
+ARG COMMIT
+RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+
+ENTRYPOINT ["./run.sh"]

docker/Dockerfile.amd64

@@ -1,20 +0,0 @@
-FROM alpine:3.9
-
-# Copy files
-ADD . /app
-RUN cp /app/data/configuration.yaml /app
-RUN cp /app/docker/run.sh /app
-RUN chmod +x /app/run.sh
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
-
-# Install dependencies
-RUN apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-    npm install --unsafe-perm && \
-    apk del make gcc g++ python linux-headers udev git
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm32v6

@@ -1,22 +0,0 @@
-FROM arm32v6/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-arm-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-arm-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-arm-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm64v8

@@ -1,22 +0,0 @@
-FROM arm64v8/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-aarch64-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docs/_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-cayman

docs/index.md

@@ -1 +0,0 @@
-Moved to [https://www.zigbee2mqtt.io](https://www.zigbee2mqtt.io)

images/logo_vector.svg

@@ -0,0 +1,146 @@
+<svg class="thumb" width="1000" height="1000" viewBox="0,0,600,600" xmlns="http://www.w3.org/2000/svg" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ns1="http://www.inkscape.org/namespaces/inkscape" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+  <defs></defs>
+  <g transform="matrix(1.8288,0,0,1.8288,-76.3478,13.5303)">
+    <g transform="matrix(1.6652,0,0,1.6704,50.2843,0.6537)">
+      <svg width="186.77" height="186.769" viewBox="-5,-5,186.77,186.769">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk4">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g>
+          <polygon points="51.774,176.769 0,124.995 0,51.774 51.774,0 124.995,0 176.77,51.774 176.77,124.995   124.995,176.769 " fill="url('#linearGradientSk1zes2vk4')"></polygon>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6204,0,0,1.6204,53.946,5.3)">
+      <svg width="186.77" height="186.769" viewBox="-5,-5,186.77,186.769">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vka">
+            <stop offset="0%" stop-color="#ffc038"></stop>
+            <stop offset="100%" stop-color="#ffc038"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g>
+          <polygon points="51.774,176.769 0,124.995 0,51.774 51.774,0 124.995,0 176.77,51.774 176.77,124.995   124.995,176.769 " fill="url('#linearGradientSk1zes2vka')"></polygon>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,185.8484,123.0829)">
+      <svg width="27.265" height="16.606891038197716" viewBox="36.308,71.828,27.265,16.606891038197716">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vkg">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M41.308,76.828c1.197,2.07,2.931,3.953,5.371,5.497c0,0,1.453,1.163,3.373,1.108c1.918,0.051,3.375-1.107,3.375-1.107   c2.289-1.446,3.958-3.192,5.146-5.112c-2.274,0.496-4.875,0.754-7.556,0.754C47.714,77.968,44.316,77.583,41.308,76.828z" fill="url('#linearGradientSk1zes2vkg')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,182.6191,109.0829)">
+      <svg width="31.991999999999997" height="16.800092191011956" viewBox="34.051,63.36,31.991999999999997,16.800092191011956">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vkm">
+            <stop offset="0%" stop-color="#ffffff"></stop>
+            <stop offset="100%" stop-color="#ffffff"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M39.051,68.36c0.055,1.736,0.307,3.47,0.818,5.141c5.94,2.063,15.1,2.206,20.244,0.407c0.563-1.692,0.85-3.453,0.93-5.221   c-2.995,0.938-6.551,1.364-10.114,1.364C46.659,70.052,42.394,69.434,39.051,68.36z" fill="url('#linearGradientSk1zes2vkm')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,182.6191,96.0829)">
+      <svg width="31.867999999999995" height="16.630912092367097" viewBox="34.128,55.641000000000005,31.867999999999995,16.630912092367097">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vks">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M50.929,62.06c-3.902,0-7.81-0.511-11.006-1.419c-0.373,1.537-0.658,3.156-0.795,4.812   c5.78,2.114,15.619,2.598,21.868,0.309c-0.119-1.662-0.384-3.297-0.745-4.846C57.427,61.692,54.187,62.06,50.929,62.06z" fill="url('#linearGradientSk1zes2vks')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,185.041,84.8902)">
+      <svg width="28.836000000000006" height="15.788046460420631" viewBox="35.678,48.491,28.836000000000006,15.788046460420631">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vky">
+            <stop offset="0%" stop-color="#ffffff"></stop>
+            <stop offset="100%" stop-color="#ffffff"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M59.514,58.241c-0.571-1.815-1.232-3.435-1.862-4.75c-2.599,0.543-5.42,0.809-8.212,0.809   c-2.458,0-4.881-0.204-7.097-0.579c-0.564,1.205-1.146,2.646-1.665,4.246C46.062,59.501,53.848,59.813,59.514,58.241z" fill="url('#linearGradientSk1zes2vky')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,120.4554,23.9828)">
+      <svg width="108.32981691822947" height="57.54144295152284" viewBox="-4.359346882061053,10.889,108.32981691822947,57.54144295152284">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk14">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M58.786,38.047c-0.651-1.481-1.406-2.589-1.406-2.589c-0.11-0.123-0.253-0.232-0.394-0.342  c0.422-1.017,0.659-2.14,0.659-3.324c0-3.374-1.896-6.271-4.6-7.523l2.116-7.369l6.143-0.228c0.215-0.007,0.382-0.188,0.378-0.405  c-0.008-0.215-0.19-0.375-0.406-0.378l-6.396,0.238c-0.152,0.006-0.287,0.095-0.346,0.239l-3.563,7.305  c-0.302-0.039-0.607-0.065-0.92-0.065c-0.314,0-0.621,0.026-0.924,0.066l-3.565-7.308c-0.058-0.138-0.193-0.23-0.349-0.237  l-6.393-0.236c-0.216-0.001-0.396,0.161-0.407,0.376c-0.006,0.218,0.163,0.398,0.38,0.405l6.143,0.23l2.118,7.368  c-2.703,1.252-4.598,4.147-4.597,7.52c0,1.184,0.239,2.307,0.659,3.323c-0.148,0.11-0.288,0.223-0.393,0.344  c0,0-0.739,1.108-1.38,2.663C29.184,41.662-2.639,46.12,0.916,55.936c3.944,10.877,25.455,7.546,29.052,4.598  c5.007-4.104,9.231-7.95,11.843-11c0.224,0.512,0.459,0.979,0.694,1.396c4.583,0.822,10.289,0.82,14.998-0.245  c0.166-0.346,0.332-0.731,0.496-1.137c2.625,3.011,6.768,6.769,11.648,10.77c3.596,2.949,25.106,6.275,29.048-4.599  C102.202,46.038,71.295,41.566,58.786,38.047z" fill="url('#linearGradientSk1zes2vk14')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(0.5515,-0.0096,0.0096,0.5515,184.0296,161.5473)">
+      <svg width="91.9" height="160" style="overflow: visible;">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk1g">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g transform="matrix(1,0,0,1,0,75)">
+          <path d="M 28.2-77.4L 28.2-77.4L 28.2 5.4Q 28.2 7.3  27.35 8.15Q 26.5 9  24.6 9L 24.6 9L 10.1 9Q 8.2 9  7.35 8.15Q 6.5 7.3  6.5 5.4L 6.5 5.4L 6.5-77.4Q 6.5-79.3  7.35-80.15Q 8.2-81  10.1-81L 10.1-81L 24.6-81Q 26.5-81  27.35-80.15Q 28.2-79.3  28.2-77.4ZM 85.39999999999998-77.4L 85.39999999999998-77.4L 85.39999999999998 5.4Q 85.39999999999998 7.3  84.54999999999998 8.15Q 83.69999999999999 9  81.79999999999998 9L 81.79999999999998 9L 67.29999999999998 9Q 65.39999999999998 9  64.54999999999998 8.15Q 63.69999999999998 7.3  63.69999999999998 5.4L 63.69999999999998 5.4L 63.69999999999998-77.4Q 63.69999999999998-79.3  64.54999999999998-80.15Q 65.39999999999998-81  67.29999999999998-81L 67.29999999999998-81L 81.79999999999998-81Q 83.69999999999999-81  84.54999999999998-80.15Q 85.39999999999998-79.3  85.39999999999998-77.4Z" data-textpath="| |" data-text-anchor="middle" data-font-size="100" data-letterspacing="0" data-side="left" data-offset="0.5" data-dx="0" data-dy="0" data-xmlspace="preserve" data-href="" data-font="Bungee" data-font-src="/Content/clientfonts/bungee.woff" data-font-scale="contain" transform="translate(0,0)" data-x="0" data-y="0" fill="url('#linearGradientSk1zes2vk1g')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(0.5,0,0,0.5,136.759,231)">
+      <svg width="289.30000000000007" height="160" style="overflow: visible;">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk1j">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g transform="matrix(1,0,0,1,0,75)">
+          <path d="M 25.4 0L 25.4 0L 10.5 0Q 8.6 0  7.75-0.85Q 6.9-1.7  6.9-3.6L 6.9-3.6L 6.9-68.4Q 6.9-70.3  7.75-71.15Q 8.6-72  10.5-72L 10.5-72L 26.6-72Q 28.2-72  29.25-71.3Q 30.3-70.6  31.2-69L 31.2-69L 40.2-52.7Q 40.7-51.8  41.05-51.4Q 41.4-51  42-51L 42-51L 42.9-51Q 43.5-51  43.85-51.4Q 44.2-51.8  44.7-52.7L 44.7-52.7L 53.6-69Q 54.5-70.6  55.55-71.3Q 56.6-72  58.2-72L 58.2-72L 74.4-72Q 76.3-72  77.15-71.15Q 78-70.3  78-68.4L 78-68.4L 78-3.6Q 78-1.7  77.15-0.85Q 76.3 0  74.4 0L 74.4 0L 59 0Q 57.1 0  56.25-0.85Q 55.4-1.7  55.4-3.6L 55.4-3.6L 55.4-38.8L 49.5-27.9Q 48.6-26.2  47.35-25.45Q 46.1-24.7  44.2-24.7L 44.2-24.7L 40.2-24.7Q 38.3-24.7  37.05-25.45Q 35.8-26.2  34.9-27.9L 34.9-27.9L 29-38.8L 29-3.6Q 29-1.7  28.15-0.85Q 27.3 0  25.4 0ZM 129.4 9L 129.4 9L 114 9Q 112.10000000000001 9  111.25 8.15Q 110.4 7.3  110.4 5.4L 110.4 5.4L 110.4-0.1Q 98.80000000000001-1.8  94.75-7.05Q 90.7-12.3  90.7-19.8L 90.7-19.8L 90.7-53Q 90.7-57.6  92.05000000000001-61.35Q 93.4-65.1  96.85000000000001-67.8Q 100.30000000000001-70.5  106.4-72Q 112.5-73.5  121.80000000000001-73.5L 121.80000000000001-73.5Q 131.10000000000002-73.5  137.15-72Q 143.2-70.5  146.65-67.8Q 150.10000000000002-65.1  151.45-61.35Q 152.8-57.6  152.8-53L 152.8-53L 152.8-19.8Q 152.8-16  151.9-12.75Q 151-9.5  148.8-7Q 146.60000000000002-4.5  142.75-2.7Q 138.9-0.9  133-0.1L 133-0.1L 133 5.4Q 133 7.3  132.15 8.15Q 131.3 9  129.4 9ZM 113.4-49.3L 113.4-22Q 113.4-20  114.9-18.65Q 116.4-17.3  121.80000000000001-17.3L 121.80000000000001-17.3Q 127.2-17.3  128.7-18.65Q 130.2-20  130.2-22L 130.2-22L 130.2-49.3Q 130.2-51.2  128.7-52.55Q 127.2-53.9  121.80000000000001-53.9L 121.80000000000001-53.9Q 116.4-53.9  114.9-52.55Q 113.4-51.2  113.4-49.3L 113.4-49.3ZM 199 0L 199 0L 182.79999999999998 0Q 180.9 0  180.04999999999998-0.85Q 179.2-1.7  179.2-3.6L 179.2-3.6L 179.2-52.7L 164.5-52.7Q 162.6-52.7  161.75-53.55Q 160.9-54.4  160.9-56.3L 160.9-56.3L 160.9-68.4Q 160.9-70.3  161.75-71.15Q 162.6-72  164.5-72L 164.5-72L 217.3-72Q 219.2-72  220.05-71.15Q 220.89999999999998-70.3  220.89999999999998-68.4L 220.89999999999998-68.4L 220.89999999999998-56.3Q 220.89999999999998-54.4  220.05-53.55Q 219.2-52.7  217.3-52.7L 217.3-52.7L 202.6-52.7L 202.6-3.6Q 202.6-1.7  201.75-0.85Q 200.89999999999998 0  199 0ZM 264.6 0L 264.6 0L 248.40000000000003 0Q 246.50000000000006 0  245.65000000000003-0.85Q 244.80000000000004-1.7  244.80000000000004-3.6L 244.80000000000004-3.6L 244.80000000000004-52.7L 230.10000000000005-52.7Q 228.20000000000005-52.7  227.35000000000005-53.55Q 226.50000000000006-54.4  226.50000000000006-56.3L 226.50000000000006-56.3L 226.50000000000006-68.4Q 226.50000000000006-70.3  227.35000000000005-71.15Q 228.20000000000005-72  230.10000000000005-72L 230.10000000000005-72L 282.90000000000003-72Q 284.80000000000007-72  285.65000000000003-71.15Q 286.50000000000006-70.3  286.50000000000006-68.4L 286.50000000000006-68.4L 286.50000000000006-56.3Q 286.50000000000006-54.4  285.65000000000003-53.55Q 284.80000000000007-52.7  282.90000000000003-52.7L 282.90000000000003-52.7L 268.20000000000005-52.7L 268.20000000000005-3.6Q 268.20000000000005-1.7  267.35-0.85Q 266.50000000000006 0  264.6 0Z" data-textpath="MQTT" data-text-anchor="middle" data-font-size="100" data-letterspacing="0" data-side="left" data-offset="0.5" data-dx="0" data-dy="0" data-xmlspace="preserve" data-href="" data-font="Bungee" data-font-src="/Content/clientfonts/bungee.woff" data-font-scale="contain" transform="translate(0,0)" data-x="0" data-y="0" fill="url('#linearGradientSk1zes2vk1j')"></path>
+        </g>
+      </svg>
+    </g>
+  </g>
+</svg>

index.js

@@ -18,6 +18,6 @@ let stopping = false;
 function handleQuit() {
     if (!stopping) {
         stopping = true;
-        controller.stop(() => process.exit());
+        controller.stop();
     }
 }

lib/controller.js

@@ -1,102 +1,136 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
+const fs = require('fs');
+const data = require('./util/data');
+const path = require('path');
 
 // Extensions
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/softReset');
-const ExtensionXiaomi = require('./extension/xiaomi');
-const ExtensionDevicePublish = require('./extension/devicePublish');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
-const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
-const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
-const ExtensionDeviceBind = require('./extension/deviceBind');
-const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionLivolo = require('./extension/livolo');
-const ExtensionResponder = require('./extension/responder');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/report');
+const ExtensionOnEvent = require('./extension/onEvent');
+const ExtensionOTAUpdate = require('./extension/otaUpdate');
 
 class Controller {
     constructor() {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
-        this.state = new State();
+        this.eventBus = new EventBus();
+        this.state = new State(this.eventBus);
 
-        // Bind methods
-        this.onMQTTConnected = this.onMQTTConnected.bind(this);
-        this.onZigbeeMessage = this.onZigbeeMessage.bind(this);
-        this.onMQTTMessage = this.onMQTTMessage.bind(this);
         this.publishEntityState = this.publishEntityState.bind(this);
+        this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceGroupMembership(this.zigbee, this.mqtt, this.publishEntityState),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDevicePublish(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionXiaomi(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionResponder(this.zigbee, this.mqtt, this.state, this.publishEntityState),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
         ];
 
+        if (settings.get().experimental.new_api) {
+            this.extensions.push(new ExtensionBridge(...args));
+        }
+
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionReport(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
+        /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionAvailability(...args));
         }
 
-        if (settings.get().experimental.livolo) {
-            // https://github.com/Koenkk/zigbee2mqtt/issues/592
-            this.extensions.push(new ExtensionLivolo(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+        const extensionPath = data.joinPath('extension');
+        if (fs.existsSync(extensionPath)) {
+            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
+            for (const extension of extensions) {
+                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
+                this.extensions.push(new Extension(...args, settings, logger));
+            }
         }
     }
 
-    onMQTTConnected() {
-        // Resend all cached states.
-        this.sendAllCachedStates();
+    async start() {
+        const settingsErrors = settings.validate();
+        if (settingsErrors) {
+            logger.error(`Refusing to start, configuration.yaml is not valid, found the following errors:`);
+            for (const error of settingsErrors) {
+                logger.error(`\t - ${error}`);
+            }
+            logger.error(
+                `If you don't know how to solve this, read https://www.zigbee2mqtt.io/configuration/configuration.html`,
+            );
+            process.exit(1);
+        }
 
-        // Call extensions
-        this.extensions.filter((e) => e.onMQTTConnected).forEach((e) => e.onMQTTConnected());
-    }
+        this.state.start();
 
-    onZigbeeStarted() {
-        // Log zigbee clients on startup and configure.
-        const devices = this.zigbee.getAllClients();
+        const info = await utils.getZigbee2mqttVersion();
+        logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
+
+        // Start zigbee
+        try {
+            await this.zigbee.start();
+            this.callExtensionMethod('onZigbeeStarted', []);
+            this.zigbee.on('event', this.onZigbeeEvent.bind(this));
+            this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
+        } catch (error) {
+            logger.error('Failed to start zigbee');
+            logger.error('Exiting...');
+            logger.error(error.stack);
+            process.exit(1);
+        }
+
+        // Log zigbee clients on startup
+        const devices = this.zigbee.getClients();
         logger.info(`Currently ${devices.length} devices are joined:`);
-        devices.forEach((device) => {
-            logger.info(this.getDeviceStartupLogMessage(device));
-        });
+        for (const device of devices) {
+            const entity = this.zigbee.resolveEntity(device);
+            logger.info(
+                (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
+                ` (${entity.device.ieeeAddr}): ` +
+                (entity.definition ?
+                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
+                    'Not supported ') +
+                `(${entity.device.type})`,
+            );
+        }
 
         // Enable zigbee join.
         if (settings.get().permit_join) {
@@ -105,169 +139,196 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        this.zigbee.permitJoin(settings.get().permit_join);
+        await this.zigbee.permitJoin(settings.get().permit_join);
 
-        // Connect to MQTT broker
-        this.mqtt.connect(this.onMQTTMessage, this.onMQTTConnected);
+        // MQTT
+        this.mqtt.on('message', this.onMQTTMessage.bind(this));
+        await this.mqtt.connect();
 
-        // Call extensions
-        this.extensions.filter((e) => e.onZigbeeStarted).forEach((e) => e.onZigbeeStarted());
-    }
-
-    onZigbeeMessage(message) {
-        // Variables
-        let device = null;
-        let mappedDevice = null;
-
-        // Check if message has a device
-        if (message.endpoints && message.endpoints.length && message.endpoints[0].device) {
-            device = message.endpoints[0].device;
-        }
-
-        // Retrieve modelId from message
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        }
-
-        // Log
-        logger.debug(
-            `Received zigbee message of type '${message.type}' with data '${JSON.stringify(message.data)}'` +
-            (device ? ` of device '${device.modelId}' (${device.ieeeAddr})` : '') +
-            (message.groupid ? ` with groupID ${message.groupID}` : '') +
-            (message.endpoints && message.endpoints.length ? ` of endpoint ${message.endpoints[0].epId}` : '')
-        );
-
-        // Call extensions.
-        this.extensions
-            .filter((e) => e.onZigbeeMessage)
-            .forEach((e) => e.onZigbeeMessage(message, device, mappedDevice));
-    }
-
-    onMQTTMessage(topic, message) {
-        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
-
-        // Call extensions
-        const results = this.extensions
-            .filter((e) => e.onMQTTMessage)
-            .map((e) => e.onMQTTMessage(topic, message));
-
-        if (!results.includes(true)) {
-            logger.warn(`Cannot handle MQTT message on '${topic}' with data '${message}'`);
-        }
-    }
-
-    start() {
-        logger.info(`Logging to directory: '${logger.directory}'`);
-
-        logger.cleanup();
-
-        this.state.start();
-
-        this.startupLogVersion(() => {
-            this.zigbee.start(this.onZigbeeMessage, (error) => {
-                if (error) {
-                    logger.error('Failed to start', error);
-                    logger.error('Exiting...');
-                    process.exit(1);
-                } else {
-                    this.onZigbeeStarted();
+        // Send all cached states.
+        if (settings.get().advanced.cache_state) {
+            for (const device of this.zigbee.getClients()) {
+                if (this.state.exists(device.ieeeAddr)) {
+                    this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
                 }
-            });
-        });
+            }
+        }
+
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
+
+        // Call extensions
+        await this.callExtensionMethod('onMQTTConnected', []);
     }
 
-    stop(callback) {
+    async stop() {
         // Call extensions
-        this.extensions.filter((e) => e.stop).forEach((e) => e.stop());
+        await this.callExtensionMethod('stop', []);
 
         // Wrap-up
         this.state.stop();
+        await this.mqtt.disconnect();
 
-        this.mqtt.disconnect();
-        this.zigbee.stop(callback);
+        try {
+            await this.zigbee.stop();
+            process.exit(0);
+        } catch (error) {
+            logger.error('Failed to stop zigbee');
+            process.exit(1);
+        }
     }
 
-    startupLogVersion(callback) {
-        utils.getZigbee2mqttVersion((info) => {
-            logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
-            callback();
-        });
+    async onZigbeeAdapterDisconnected() {
+        logger.error('Adapter disconnected, stopping');
+        await this.stop();
     }
 
-    getDeviceStartupLogMessage(device) {
-        let friendlyName = 'unknown';
-        let type = 'unknown';
-        let friendlyDevice = {model: 'unkown', description: 'unknown'};
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        if (mappedModel) {
-            friendlyDevice = mappedModel;
+    async onZigbeeEvent(type, data) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
+            // Only deviceLeave doesn't have a device (not interesting to add to settings)
+            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
         }
 
-        if (settings.getDevice(device.ieeeAddr)) {
-            friendlyName = settings.getDevice(device.ieeeAddr).friendly_name;
-        }
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
 
-        if (device.type) {
-            type = utils.correctDeviceType(device);
-        }
+        if (type === 'message') {
+            logger.debug(
+                `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
+                `, data '${JSON.stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
+            );
+        } else if (type === 'deviceJoined') {
+            logger.info(`Device '${name}' joined`);
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
 
-        return `${friendlyName} (${device.ieeeAddr}): ${friendlyDevice.model} - ` +
-            `${friendlyDevice.vendor} ${friendlyDevice.description} (${type})`;
-    }
-
-    sendAllCachedStates() {
-        this.zigbee.getAllClients().forEach((device) => {
-            if (this.state.exists(device.ieeeAddr)) {
-                this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
+                } else {
+                    logger.warn(
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
+                    );
+                }
+            } else if (data.status === 'failed') {
+                logger.error(`Failed to interview '${name}', device has not successfully been paired`);
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    logger.info(`Starting interview of '${name}'`);
+                }
             }
-        });
+        } else if (type === 'deviceAnnounce') {
+            logger.debug(`Device '${name}' announced itself`);
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+            }
+        }
+
+        // Call extensions
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
     }
 
-    publishEntityState(entityID, payload) {
-        const entity = settings.resolveEntity(entityID);
-        const appSettings = settings.get();
-        let messagePayload = {...payload};
+    onMQTTMessage(payload) {
+        const {topic, message} = payload;
+        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
 
-        const currentState = this.state.exists(entityID) ? this.state.get(entityID) : {};
+        // Call extensions
+        this.callExtensionMethod('onMQTTMessage', [topic, message]);
+    }
+
+    async publishEntityState(IDorName, payload, stateChangeReason=null) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
+            logger.error(`'${IDorName}' does not exist, skipping publish`);
+            return;
+        }
+
+        let messagePayload = {...payload};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
         const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(entityID, newState);
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
             messagePayload = newState;
         }
 
-        const entitySettings = entity.type === 'device' ? settings.getDevice(entityID) : settings.getGroup(entityID);
+        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: entitySettings ? entitySettings.retain : false,
-            qos: entitySettings && entitySettings.qos ? entitySettings.qos : 0,
+            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
+            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
         };
 
-        if (entity.type === 'device' && appSettings.mqtt.include_device_information) {
-            messagePayload.device = this.getDeviceInfoForMqtt(entityID);
+        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention};
+        }
+
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
+            const attributes = [
+                'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
+                'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
+            ];
+
+            messagePayload.device = {
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
+            };
+
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
+        // filter mqtt message attributes
+        if (deviceOptions.filtered_attributes) {
+            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
         if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'json') {
-                this.mqtt.publish(entity.friendlyName, JSON.stringify(messagePayload), options);
-            } else if (settings.get().experimental.output === 'attribute') {
-                this.iteratePayloadForAttrOutput(entity.friendlyName+'/', messagePayload, options);
+            if (settings.get().experimental.output === 'attribute_and_json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+            } else if (settings.get().experimental.output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+            } else {
+                /* istanbul ignore else */
+                if (settings.get().experimental.output === 'attribute') {
+                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+                }
             }
         }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
     }
 
-    iteratePayloadForAttrOutput(topicRoot, payload, options) {
-        Object.keys(payload).forEach((key) => {
-            let subPayload = payload[key];
+    async iteratePayloadAttributeOutput(topicRoot, payload, options) {
+        for (const [key, value] of Object.entries(payload)) {
+            let subPayload = value;
             let message;
 
             // Special cases
-            if (key === 'color' &&
-                subPayload.r !== undefined &&
-                subPayload.g !== undefined &&
-                subPayload.b !== undefined) {
+            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
                 subPayload = [subPayload.r, subPayload.g, subPayload.b];
             }
 
@@ -275,36 +336,30 @@ class Controller {
             if (Array.isArray(subPayload)) {
                 message = subPayload.map((x) => `${x}`).join(',');
             } else if (typeof subPayload === 'object') {
-                return this.iteratePayloadForAttrOutput(topicRoot+key+'-', subPayload, options);
+                return this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
             } else {
                 message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
             }
 
-            this.mqtt.publish(`${topicRoot}${key}`, message, options);
-        });
+            await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+        }
     }
 
-    getDeviceInfoForMqtt(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        const {
-            type, nwkAddr, manufId, manufName, powerSource, modelId, hwVersion, swBuildId, status, dateCode,
-        } = device;
-        const deviceSettings = settings.getDevice(device.ieeeAddr);
-
-        return {
-            ieeeAddr,
-            friendlyName: deviceSettings ? (deviceSettings.friendly_name || '') : '',
-            type,
-            nwkAddr,
-            manufId,
-            manufName,
-            powerSource,
-            modelId,
-            hwVersion: hwVersion ? hwVersion : 'unknown',
-            swBuildId: swBuildId ? swBuildId : 'unknown',
-            dateCode: dateCode ? dateCode : 'unknown',
-            status,
-        };
+    async callExtensionMethod(method, parameters) {
+        for (const extension of this.extensions) {
+            if (extension[method]) {
+                try {
+                    await extension[method](...parameters);
+                } catch (error) {
+                    /* istanbul ignore next */
+                    logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
+                    /* istanbul ignore next */
+                    if (process.env.JEST_WORKER_ID !== undefined) {
+                        throw error;
+                    }
+                }
+            }
+        }
     }
 }
 

lib/eventBus.js

@@ -0,0 +1,24 @@
+const events = require('events');
+const assert = require('assert');
+
+const allowedEvents = [
+    'deviceRemoved', // Device has been removed
+    'deviceRenamed', // Device has been renamed
+    'groupRenamed', // Group has been renamed
+    'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
+];
+
+class EventBus extends events.EventEmitter {
+    emit(event, data) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.emit(event, data);
+    }
+
+    on(event, callback) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.on(event, callback);
+    }
+}
+
+module.exports = EventBus;

lib/extension/availability.js

@@ -0,0 +1,195 @@
+const logger = require('../util/logger');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const Extension = require('./extension');
+
+// Pingable end devices, some end devices should be pinged
+// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
+];
+
+const Hours25 = 1000 * 60 * 60 * 25;
+
+/**
+ * This extensions pings devices to check if they are online.
+ */
+class Availability extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.availability_timeout = settings.get().advanced.availability_timeout;
+        this.timers = {};
+        this.state = {};
+
+        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
+        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
+    }
+
+    inWhitelistOrNotInBlacklist(device) {
+        const ieeeAddr = device.ieeeAddr;
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings.friendlyName;
+
+        // Whitelist is not empty and device is in it, enable availability
+        if (this.whitelist.length > 0) {
+            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        }
+
+        // Device is on blacklist, disable availability
+        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
+            return false;
+        }
+
+        return true;
+    }
+
+    isPingable(device) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
+            return true;
+        }
+
+        // Device is a mains powered router
+        return utils.isRouter(device) && !utils.isBatteryPowered(device);
+    }
+
+    onMQTTConnected() {
+        for (const device of this.zigbee.getClients()) {
+            // Mark all devices as online on start
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
+
+            if (this.inWhitelistOrNotInBlacklist(device)) {
+                if (this.isPingable(device)) {
+                    this.setTimerPingable(device);
+                } else {
+                    this.timers[ieeeAddr] = setInterval(() => {
+                        this.handleIntervalNotPingable(device);
+                    }, utils.secondsToMilliseconds(300));
+                }
+            }
+        }
+    }
+
+    async handleIntervalPingable(device) {
+        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
+            logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
+            return;
+        }
+
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
+        try {
+            await device.ping();
+            this.publishAvailability(device, true);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
+        } catch (error) {
+            this.publishAvailability(device, false);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
+        } finally {
+            this.setTimerPingable(device);
+        }
+    }
+
+    async handleIntervalNotPingable(device) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
+            return;
+        }
+
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
+
+        if (ago > Hours25) {
+            this.publishAvailability(device, false);
+        }
+    }
+
+    setTimerPingable(device) {
+        if (this.timers[device.ieeeAddr]) {
+            clearTimeout(this.timers[device.ieeeAddr]);
+        }
+
+        this.timers[device.ieeeAddr] = setTimeout(async () => {
+            await this.handleIntervalPingable(device);
+        }, utils.secondsToMilliseconds(this.availability_timeout));
+    }
+
+    async stop() {
+        for (const timer of Object.values(this.timers)) {
+            clearTimeout(timer);
+        }
+
+        this.zigbee.getClients().forEach((device) => this.publishAvailability(device, false));
+    }
+
+    async onReconnect(device) {
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            const used = [];
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter && !used.includes(converter)) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                        used.push(converter);
+                    }
+                }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
+            }
+        }
+    }
+
+    publishAvailability(device, available, force=false) {
+        const ieeeAddr = device.ieeeAddr;
+        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
+            this.onReconnect(device);
+        }
+
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
+        const topic = `${name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        if (this.state[ieeeAddr] !== available || force) {
+            this.state[ieeeAddr] = available;
+            this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+        if (!device) {
+            return;
+        }
+
+        if (this.inWhitelistOrNotInBlacklist(device)) {
+            this.publishAvailability(data.device, true);
+
+            if (this.isPingable(device)) {
+                // When a zigbee message from a device is received we know the device is still alive.
+                // => reset the timer.
+                this.setTimerPingable(device);
+
+                const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
+                if (online && type === 'deviceAnnounce' && !utils.isIkeaTradfriDevice(device)) {
+                    /**
+                     * In case the device is powered off AND on within the availability timeout,
+                     * zigbee2qmtt does not detect the device as offline (device is still marked online).
+                     * When a device is turned on again the state could be out of sync.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
+                     * deviceAnnounce is typically send when a device comes online.
+                     *
+                     * This isn't needed for TRADFRI devices as they already send the state themself.
+                     */
+                    this.onReconnect(device);
+                }
+            }
+        }
+    }
+}
+
+module.exports = Availability;

lib/extension/bind.js

@@ -0,0 +1,121 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const assert = require('assert');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const Extension = require('./extension');
+
+const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+
+// See zigbee-herdsman-converters devices.js
+const defaultBindGroup = {type: 'group_number', ID: 901};
+
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
+        }
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        }
+
+        return {type, sourceKey, targetKey};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
+
+        // Find source; can only be a device and target
+        const source = this.zigbee.resolveEntity(sourceKey);
+        assert(source != null && source.type === 'device', 'Source undefined or not a device');
+        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        assert(target != null, 'Target is unknown');
+
+        const sourceName = source.settings.friendlyName;
+        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+        let attemptedToBindSomething = false;
+
+        // Find which clusters are supported by both the source and target.
+        // Groups are assumed to support all clusters.
+        for (const cluster of clusters) {
+            const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+
+            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                attemptedToBindSomething = true;
+                try {
+                    let bindTarget = null;
+                    if (target.type === 'group') bindTarget = target.group;
+                    else if (target.type === 'group_number') bindTarget = target.ID;
+                    else bindTarget = target.endpoint;
+
+                    if (type === 'bind') {
+                        await source.endpoint.bind(cluster, bindTarget);
+                    } else {
+                        await source.endpoint.unbind(cluster, bindTarget);
+                    }
+
+                    logger.info(
+                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                        `'${sourceName}' to '${targetName}'`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
+                } catch (error) {
+                    logger.error(
+                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                        `'${targetName}' (${error})`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}_failed`,
+                                message: {from: sourceName, to: targetName, cluster}}),
+                        );
+                    }
+                }
+            }
+        }
+
+        if (!attemptedToBindSomething) {
+            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                );
+            }
+        }
+    }
+}
+
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,211 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+
+        this.requestLookup = {
+            'permitjoin': this.permitJoin.bind(this),
+            // 'device/remove': this.deviceRemove.bind(this),
+            // 'device/forceremove': this.deviceForceRemove.bind(this),
+            // 'device/ban': this.deviceBan.bind(this),
+            // 'group/remove': this.groupRemove.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = JSON.parse(message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
+            else if (type === 'deviceInterview') {
+                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
+                if (data.status === 'successful') {
+                    payload.supported = !!resolvedEntity.definition;
+                    payload.definition = resolvedEntity.definition ? {
+                        model: resolvedEntity.definition.model,
+                        vendor: resolvedEntity.definition.vendor,
+                        description: resolvedEntity.definition.description,
+                        supports: resolvedEntity.definition.supports,
+                    } : null;
+                }
+            } else payload = {ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    // async deviceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'device', message);
+    // }
+
+    // async deviceForceRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
+    // }
+
+    // async deviceBan(message) {
+    //     return this.removeForceRemoveOrBanEntity('ban', 'device', message);
+    // }
+
+    // async groupRemove(message) {
+    //     return this.removeForceRemoveOrBanEntity('remove', 'group', message);
+    // }
+
+    async permitJoin(message) {
+        const value = typeof message === 'object' ? message.value : message;
+        await this.zigbee.permitJoin(value);
+        await this.publishInfo();
+        return utils.getResponse(message, {value: value}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    // async removeForceRemoveOrBanEntity(action, entityType, message) {
+    //     const ID = typeof message === 'object' ? message.ID : message.trim();
+    //     const entity = this.zigbee.resolveEntity(ID);
+    //     if (!entity || entity.type !== entityType) {
+    //         throw new Error(`${ID} is not a ${entityType}`);
+    //     }
+
+    //     const lookup = {
+    //         ban: ['banned', 'Banning', 'ban'],
+    //         force_remove: ['force_removed', 'Force removing', 'force remove'],
+    //         remove: ['removed', 'Removing', 'remove'],
+    //     };
+
+    //     try {
+    //         logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+    //         if (entity.type === 'device') {
+    //             if (action === 'ban') {
+    //                 settings.banDevice(entity.settings.ID);
+    //             }
+
+    //             action === 'force_remove' ?
+    //                 await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
+    //         } else {
+    //             await entity.group.removeFromDatabase();
+    //         }
+
+    //         // Fire event
+    //         if (entity.type === 'device') {
+    //             this.eventBus.emit('deviceRemoved', {device: entity.device});
+    //         }
+
+    //         // Remove from configuration.yaml
+    //         entity.type === 'device' ?
+    //             settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+
+    //         // Remove from state
+    //         this.state.remove(entity.settings.ID);
+
+    //         logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+    //         entity.type === 'device' ? this.publishDevices() : this.publishGroups();
+    //         return utils.getResponse(message, {ID}, null);
+    //     } catch (error) {
+    //         throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+    //     }
+    // }
+
+    async publishInfo() {
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            logLevel: logger.getLevel(),
+            permitJoin: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    async publishDevices(topic, message) {
+        const devices = this.zigbee.getClients().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const definitionPayload = definition ? {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                supports: definition.supports,
+            } : null;
+
+            return {
+                ieeeAddress: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+                supported: !!definition,
+                friendlyName: resolved.name,
+                definition: definitionPayload,
+                powerSource: device.powerSource,
+                softwareBuildID: device.softwareBuildID,
+                dateCode: device.dateCode,
+                interviewing: device.interviewing,
+                interviewCompleted: device.interviewCompleted,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+    }
+
+    async publishGroups(topic, message) {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                ID: group.groupID,
+                friendlyName: resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieeeAddress: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+    }
+}
+
+module.exports = Bridge;

lib/extension/bridgeConfig.js

@@ -1,289 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const utils = require('../util/utils');
-
-const configRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+))`);
-const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
-
-class BridgeConfig {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-
-        // Bind functions
-        this.permitJoin = this.permitJoin.bind(this);
-        this.lastSeen = this.lastSeen.bind(this);
-        this.elapsed = this.elapsed.bind(this);
-        this.reset = this.reset.bind(this);
-        this.logLevel = this.logLevel.bind(this);
-        this.devices = this.devices.bind(this);
-        this.groups = this.groups.bind(this);
-        this.rename = this.rename.bind(this);
-        this.remove = this.remove.bind(this);
-        this.ban = this.ban.bind(this);
-        this.deviceOptions = this.deviceOptions.bind(this);
-        this.addGroup = this.addGroup.bind(this);
-        this.removeGroup = this.removeGroup.bind(this);
-        this.whitelist= this.whitelist.bind(this);
-
-        // Set supported options
-        this.supportedOptions = {
-            'permit_join': this.permitJoin,
-            'last_seen': this.lastSeen,
-            'elapsed': this.elapsed,
-            'reset': this.reset,
-            'log_level': this.logLevel,
-            'devices': this.devices,
-            'groups': this.groups,
-            'devices/get': this.devices,
-            'rename': this.rename,
-            'remove': this.remove,
-            'ban': this.ban,
-            'device_options': this.deviceOptions,
-            'add_group': this.addGroup,
-            'remove_group': this.removeGroup,
-            'whitelist': this.whitelist,
-        };
-    }
-
-    whitelist(topic, message) {
-        message = message.toString();
-        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
-        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-        const whitelisted = settings.whitelistDevice(deviceID);
-        whitelisted ? logger.info(`Whitelisted '${deviceID}'`) : logger.error(`Failed to whitelist '${deviceID}'`);
-        this.mqtt.log('device_whitelisted', deviceID);
-    }
-
-    deviceOptions(topic, message) {
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error('Failed to parse message as JSON');
-            return;
-        }
-
-        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
-            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
-            return;
-        }
-
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(json.friendly_name);
-        if (!ieeeAddr) {
-            logger.error(`Failed to find device '${json.friendly_name}'`);
-            return;
-        }
-
-        settings.changeDeviceOptions(ieeeAddr, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
-    }
-
-    permitJoin(topic, message) {
-        this.zigbee.permitJoin(message.toString().toLowerCase() === 'true', () => this.publish());
-    }
-
-    reset(topic, message) {
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.error('Soft reset failed');
-            } else {
-                logger.info('Soft resetted ZNP');
-            }
-        });
-    }
-
-    lastSeen(topic, message) {
-        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'last_seen'], message);
-        logger.info(`Set last_seen to ${message}`);
-    }
-
-    elapsed(topic, message) {
-        const allowed = ['true', 'false'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'elapsed'], message === 'true');
-        logger.info(`Set elapsed to ${message}`);
-    }
-
-    logLevel(topic, message) {
-        const level = message.toString().toLowerCase();
-        if (allowedLogLevels.includes(level)) {
-            logger.info(`Switching log level to '${level}'`);
-            logger.transports.console.level = level;
-            logger.transports.file.level = level;
-        } else {
-            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
-        }
-
-        this.publish();
-    }
-
-    devices(topic, message) {
-        const devices = this.zigbee.getDevices().map((device) => {
-            const payload = {
-                ieeeAddr: device.ieeeAddr,
-                type: device.type,
-            };
-
-            if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelId;
-                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                payload.nwkAddr = device.nwkAddr;
-                payload.manufId = device.manufId;
-                payload.manufName = device.manufName;
-                payload.powerSource = device.powerSource;
-                payload.modelId = device.modelId;
-                payload.hwVersion = device.hwVersion;
-                payload.swBuildId = device.swBuildId;
-                payload.dateCode = device.dateCode;
-            }
-
-            return payload;
-        });
-
-        if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
-        } else {
-            this.mqtt.log('devices', devices);
-        }
-    }
-
-    groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups());
-    }
-
-    rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} ` +
-                            `got ${message.toString()}`;
-
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error(invalid);
-            return;
-        }
-
-        // Validate message
-        if (!json.new || !json.old) {
-            logger.error(invalid);
-            return;
-        }
-
-        if (settings.changeFriendlyName(json.old, json.new)) {
-            logger.info(`Successfully renamed - ${json.old} to ${json.new} `);
-            this.mqtt.log('device_renamed', {from: json.old, to: json.new});
-        } else {
-            logger.error(`Failed to renamed - ${json.old} to ${json.new}`);
-            return;
-        }
-    }
-
-    addGroup(topic, message) {
-        const name = message.toString();
-        const added = settings.addGroup(name);
-        added ? logger.info(`Added group '${name}'`) : logger.error(`Failed to add group '${name}'`);
-    }
-
-    removeGroup(topic, message) {
-        const name = message.toString();
-        const removed = settings.removeGroup(name);
-        removed ? logger.info(`Removed group '${name}'`) : logger.error(`Failed to remove group '${name}'`);
-    }
-
-    remove(topic, message) {
-        this.removeOrBan(false, message);
-    }
-
-    ban(topic, message) {
-        this.removeOrBan(true, message);
-    }
-
-    removeOrBan(ban, message) {
-        message = message.toString();
-        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
-        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-        const device = this.zigbee.getDevice(deviceID);
-
-        const cleanup = () => {
-            // Remove from configuration.yaml
-            settings.removeDevice(deviceID);
-
-            // Remove from state
-            this.state.remove(deviceID);
-
-            logger.info(`Successfully ${ban ? 'banned' : 'removed'} ${deviceID}`);
-            this.mqtt.log(ban ? 'device_banned' : 'device_removed', message);
-        };
-
-        // Remove from zigbee network.
-        if (device) {
-            this.zigbee.removeDevice(deviceID, ban, (error) => {
-                if (!error) {
-                    cleanup();
-                } else {
-                    logger.error(`Failed to ${ban ? 'ban' : 'remove'} ${deviceID}`);
-                }
-            });
-        } else {
-            cleanup();
-        }
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.publish();
-    }
-
-    onMQTTMessage(topic, message) {
-        if (!topic.match(configRegex)) {
-            return false;
-        }
-
-        const option = topic.match(configRegex)[1];
-
-        if (!this.supportedOptions.hasOwnProperty(option)) {
-            return false;
-        }
-
-        this.supportedOptions[option](topic, message);
-
-        return true;
-    }
-
-    publish() {
-        utils.getZigbee2mqttVersion((info) => {
-            const topic = `bridge/config`;
-            const payload = {
-                version: info.version,
-                commit: info.commitHash,
-                coordinator: this.zigbee.getFirmwareVersion(),
-                log_level: logger.transports.console.level,
-                permit_join: this.zigbee.getPermitJoin(),
-            };
-
-            this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0}, null);
-        });
-    }
-}
-
-module.exports = BridgeConfig;

lib/extension/configure.js

@@ -0,0 +1,120 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
+        this.attempts = {};
+
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+    }
+
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
+
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (topic !== this.legacyTopic) {
+                return;
+            }
+
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
+
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(resolvedEntity, true);
+        }
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    async configure(resolvedEntity, force=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+        } catch (error) {
+            logger.error(
+                `Failed to configure '${resolvedEntity.name}', ` +
+                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
+            );
+            this.attempts[device.ieeeAddr]++;
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/deviceAvailability.js

@@ -1,182 +0,0 @@
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-
-// Some EndDevices should be pinged
-// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableDevices = [
-    zigbeeShepherdConverters.devices.find((d) => d.model === 'E11-G13'),
-];
-
-const toZigbeeCandidates = ['state'];
-
-/**
- * This extensions pings devices to check if they are online.
- */
-class DeviceAvailability {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.availability_timeout = settings.get().advanced.availability_timeout;
-        this.timers = {};
-        this.pending = [];
-        this.state = {};
-
-        // Initialize blacklist
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getIeeeAddrByFriendlyName(e) || e;
-        });
-    }
-
-    isPingable(device) {
-        logger.trace(`Checking if ${device.ieeeAddr} is pingable`);
-
-        if (this.blacklist.includes(device.ieeeAddr)) {
-            logger.trace(`${device.ieeeAddr} is not pingable because of blacklist`);
-            return false;
-        }
-
-        if (pingableDevices.find((d) => d.zigbeeModel.includes(device.modelId))) {
-            logger.trace(`${device.ieeeAddr} is pingable because in pingable devices`);
-            return true;
-        }
-
-        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
-        logger.trace(`${device.ieeeAddr} is pingable (${result}) not router or battery powered`);
-        return result;
-    }
-
-    getAllPingableDevices() {
-        return this.zigbee.getAllClients().filter((d) => this.isPingable(d));
-    }
-
-    onMQTTConnected() {
-        // As some devices are not checked for availability (e.g. battery powered devices)
-        // we mark all device as online by default.
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, true));
-
-        // Start timers for all devices
-        this.getAllPingableDevices().forEach((device) => this.setTimer(device));
-    }
-
-    handleInterval(device) {
-        // Check if a job is already pending.
-        // This avoids overflowing of the queue in case the queue is not able to catch-up with the jobs being added.
-        const ieeeAddr = device.ieeeAddr;
-        if (this.pending.includes(ieeeAddr)) {
-            logger.debug(`Skipping ping for ${ieeeAddr} becuase job is already in queue`);
-            return;
-        }
-
-        this.pending.push(ieeeAddr);
-
-        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const errorLogLevel = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
-        const mechanism = utils.isXiaomiDevice(device) ? 'basic' : 'default';
-
-        this.zigbee.ping(ieeeAddr, errorLogLevel, (error) => {
-            this.publishAvailability(ieeeAddr, !error);
-
-            // Remove from pending jobs.
-            const index = this.pending.indexOf(ieeeAddr);
-            if (index !== -1) {
-                this.pending.splice(index, 1);
-            }
-
-            this.setTimer(device);
-        }, mechanism);
-    }
-
-    setTimer(device) {
-        if (this.timers[device.ieeeAddr]) {
-            clearTimeout(this.timers[device.ieeeAddr]);
-        }
-
-        this.timers[device.ieeeAddr] = setTimeout(() => {
-            this.handleInterval(device);
-        }, utils.secondsToMilliseconds(this.availability_timeout));
-    }
-
-    stop() {
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, false));
-    }
-
-    onReconnect(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        let mappedDevice = null;
-
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        }
-
-        if (mappedDevice) {
-            const converters = mappedDevice.toZigbee.filter((tz) => {
-                return tz.key.find((k) => toZigbeeCandidates.includes(k));
-            });
-
-            converters.forEach((converter) => {
-                const convertedResults = converter.convert(null, null, null, 'get');
-                if (convertedResults && convertedResults.length) {
-                    convertedResults.forEach((converted) => {
-                        this.zigbee.publish(
-                            ieeeAddr, 'device', converted.cid, converted.cmd, converted.cmdType,
-                            converted.zclData, converted.cfg, null, () => {}
-                        );
-                    });
-                }
-            });
-        }
-    }
-
-    publishAvailability(ieeeAddr, available) {
-        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
-            this.onReconnect(ieeeAddr);
-        }
-
-        this.state[ieeeAddr] = available;
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
-        const topic = `${name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device && this.isPingable(this.zigbee.getDevice(device.ieeeAddr))) {
-            // When a zigbee message from a device is received we know the device is still alive.
-            // => reset the timer.
-            this.setTimer(device);
-
-            const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
-            const offline = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr];
-
-            if (!online && !offline) {
-                // A new device has been connected
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (offline) {
-                // When a message is received and the device is marked as offline, mark it online.
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (online) {
-                /**
-                 * In case the device is powered off AND on within the availability timeout,
-                 * zigbee2qmtt does not detect the device as offline (device is still marked online).
-                 * When a device is turned on again the state could be out of sync.
-                 * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
-                 * endDeviceAnnce is typically send when a device comes online.
-                 *
-                 * This isn't needed for TRADFRI devices as they already send the state themself.
-                 */
-                if (message.type === 'endDeviceAnnce' && !utils.isIkeaTradfriDevice(device)) {
-                    this.onReconnect(device.ieeeAddr);
-                }
-            }
-        }
-    }
-}
-
-module.exports = DeviceAvailability;

lib/extension/deviceBind.js

@@ -1,128 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-
-const allowedClusters = [
-    5, // genScenes
-    6, // genOnOff
-    8, // genLevelCtrl
-    768, // lightingColorCtrl
-];
-
-class DeviceBind {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+/+`);
-    }
-
-    getIDAndPostfix(topic) {
-        let postfix = null;
-        if (postfixes.find((p) => topic.endsWith(`/${p}`))) {
-            postfix = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-            // Remove postfix from topic
-            topic = topic.replace(`/${postfix}`, '');
-        }
-
-        return {ID: topic, postfix};
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-
-        // Parse type from topic
-        const type = topic.split('/')[0];
-
-        // Remove type from topic
-        topic = topic.replace(`${type}/`, '');
-        return {type, ...this.getIDAndPostfix(topic)};
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Find source; can only be a device.
-        const sourceEntity = settings.resolveEntity(topic.ID);
-        const source = utils.getEndpointByEntityID(this.zigbee, sourceEntity.ID, topic.postfix);
-
-        const targetEntityIDPostfix= this.getIDAndPostfix(message.toString());
-        const targetEntity = settings.resolveEntity(targetEntityIDPostfix.ID);
-        let target = null;
-        if (targetEntity.type === 'device') {
-            target = utils.getEndpointByEntityID(this.zigbee, targetEntity.ID, targetEntityIDPostfix.postfix);
-        } else if (targetEntity.type === 'group') {
-            target = targetEntity.ID;
-        }
-
-        if (!source || !target) {
-            return false;
-        }
-
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters (as we don't know which devices are in)
-        let supported = [];
-        if (targetEntity.type === 'device') {
-            supported = target.getSimpleDesc().inClusterList.filter((cluster) => {
-                return allowedClusters.includes(cluster);
-            });
-        } else if (targetEntity.type === 'group') {
-            supported = allowedClusters;
-        }
-
-        const clusters = source.getSimpleDesc().outClusterList.filter((cluster) => {
-            return supported.includes(cluster);
-        });
-
-        // Bind
-        clusters.forEach((cluster) => {
-            logger.debug(
-                `${topic.type}ing cluster '${cluster}' from '${sourceEntity.ID}' ` +
-                `(${source.epId}) to '${targetEntity.ID}'`
-            );
-
-            this.zigbee[topic.type](source, cluster, target, (error) => {
-                if (error) {
-                    logger.error(
-                        `Failed to ${topic.type} cluster '${cluster}' from '${sourceEntity.ID}' to ` +
-                        `'${targetEntity.ID}' (${error})`
-                    );
-                } else {
-                    logger.info(
-                        `Successfully ${topic.type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceEntity.ID}' to '${targetEntity.ID}'`
-                    );
-
-                    this.mqtt.log(
-                        `device_${topic.type}`,
-                        {from: sourceEntity.ID, to: targetEntity.ID, cluster}
-                    );
-                }
-            });
-        });
-
-        return true;
-    }
-}
-
-module.exports = DeviceBind;

lib/extension/deviceConfigure.js

@@ -1,89 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const Queue = require('queue');
-
-/**
- * This extensions handles configuration of devices.
- */
-class DeviceConfigure {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.configured = {};
-        this.attempts = {};
-
-        this.queue = new Queue();
-        this.queue.concurrency = 1;
-        this.queue.autostart = true;
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => {
-            const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-
-            if (mappedDevice) {
-                this.configure(device.ieeeAddr, mappedDevice);
-            }
-        });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device && mappedDevice) {
-            const ieeeAddr = device.ieeeAddr;
-            if (!this.configured[ieeeAddr]) {
-                this.configure(ieeeAddr, mappedDevice);
-            }
-        }
-    }
-
-    configure(ieeeAddr, mappedDevice) {
-        if (!this.attempts.hasOwnProperty(ieeeAddr)) {
-            this.attempts[ieeeAddr] = 0;
-        }
-
-        if (mappedDevice.configure) {
-            const friendlyName = settings.getDevice(ieeeAddr) ? settings.getDevice(ieeeAddr).friendly_name : 'unknown';
-            // mark device as currently being configured to prevent race conditions
-            this.configured[ieeeAddr] = 'configuring';
-
-            this.queue.push((queueCallback) => {
-                logger.debug(`Configuring ${friendlyName} (${ieeeAddr}) ...`);
-
-                // Call configure function of this device.
-                mappedDevice.configure(ieeeAddr, this.zigbee.shepherd, this.zigbee.getCoordinator(), (ok, msg) => {
-                    if (ok) {
-                        logger.info(`Successfully configured ${friendlyName} (${ieeeAddr})`);
-                        this.configured[ieeeAddr] = 'configured';
-                    } else {
-                        // If a device is not joined in the network the configure command may fail
-                        // (especially, immediately after zigbee start)
-                        // also, this command may fail if the network is busy or lost data packets
-                        if (this.attempts[ieeeAddr] > 1) {
-                            // Give up after 3 attempts.
-                            this.configured[ieeeAddr] = 'failed';
-                        } else {
-                            // we're not done yet so mark not busy
-                            delete this.configured[ieeeAddr];
-                        }
-
-                        logger.warn(
-                            `Failed to configure ${friendlyName} (${ieeeAddr}) ('${msg}')` +
-                            ` (attempt #${this.attempts[ieeeAddr] + 1})`
-                        );
-
-                        logger.warn(`This can be ignored if the device is working properly`);
-
-                        this.attempts[ieeeAddr] += 1;
-                    }
-
-                    queueCallback();
-                });
-            });
-        } else {
-            // no need to configure anyting. So just mark as done
-            this.configured[ieeeAddr] = 'configured';
-        }
-    }
-}
-
-module.exports = DeviceConfigure;

lib/extension/deviceGroupMembership.js

@@ -1,73 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/.+/get_group_membership$`);
-
-class DeviceGroupMembership {
-    constructor(zigbee, mqtt, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/device/+/get_group_membership`);
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/device/`, '');
-
-        // Remove command from topic
-        topic = topic.replace(`/get_group_membership`, '');
-
-        return {friendlyName: topic};
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Map message to ieeeAddr and check if device exist.
-        const friendlyName = topic.friendlyName;
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(friendlyName) || friendlyName;
-        if (!this.zigbee.getDevice(ieeeAddr)) {
-            logger.error(`Failed to find device '${friendlyName}'`);
-            return;
-        }
-
-        const callback = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to get membership of ${ieeeAddr}`);
-            } else {
-                const {grouplist, capacity} = rsp;
-                const msgGroupList = `${ieeeAddr} is in groups [${grouplist}]`;
-                let msgCapacity;
-                if (capacity === 254) {
-                    msgCapacity = 'it can be a part of at least 1 more group';
-                } else {
-                    msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
-                }
-                logger.info(`${msgGroupList} and ${msgCapacity}`);
-
-                this.publishEntityState(ieeeAddr, {group_list: grouplist, group_capacity: capacity});
-            }
-        };
-
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', 'getMembership', 'functional',
-            {groupcount: 0, grouplist: []}, null, null, callback,
-        );
-
-        return true;
-    }
-}
-
-module.exports = DeviceGroupMembership;

lib/extension/devicePublish.js

@@ -1,231 +0,0 @@
-
-const settings = require('../util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-const postfixes = utils.getPostfixes();
-const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
-
-const maxDepth = 20;
-
-const groupConverters = [
-    zigbeeShepherdConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeShepherdConverters.toZigbeeConverters.light_colortemp,
-    zigbeeShepherdConverters.toZigbeeConverters.light_color,
-    zigbeeShepherdConverters.toZigbeeConverters.light_alert,
-    zigbeeShepherdConverters.toZigbeeConverters.ignore_transition,
-];
-
-class DevicePublish {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < maxDepth; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
-    parseTopic(topic) {
-        const match = topic.match(topicRegex);
-        if (!match) {
-            return null;
-        }
-
-        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
-        // If we didn't repalce base_topic we received something we don't care about
-        if (ID === match[1] || ID.match(/bridge\/config/)) {
-            return null;
-        }
-
-        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
-    }
-
-    handlePublishError(entity, message, error) {
-        const meta = {
-            entity,
-            message: message.toString(),
-        };
-
-        this.mqtt.log('zigbee_publish_error', error.toString(), meta);
-    }
-
-    handlePublished(entity, topic, converter, converted, key, value) {
-        if (topic.type === 'set' && converted.hasOwnProperty('newState') && converted.newState) {
-            const msg = converted.newState;
-
-            if (topic.postfix) {
-                msg[`state_${topic.postfix}`] = msg.state;
-                delete msg.state;
-            }
-
-            if (settings.get().advanced.last_seen !== 'disable') {
-                msg.last_seen = utils.formatDate(Date.now(), settings.get().advanced.last_seen);
-            }
-
-            this.publishEntityState(entity.ID, msg);
-        }
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Resolve the entity
-        const entity = settings.resolveEntity(topic.ID);
-
-        // Get entity details
-        let endpoint = null;
-        let converters = null;
-        let device = null;
-        let options = {};
-
-        if (entity.type === 'device') {
-            device = this.zigbee.getDevice(entity.ID);
-            if (!device) {
-                logger.error(`Failed to find device with ieeeAddr: '${entity.ID}'`);
-                this.mqtt.log('entity_not_found', entity.ID);
-                return;
-            }
-
-            // Map device to a model
-            const model = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (!model) {
-                logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-                return;
-            }
-
-            // Determine endpoint to publish to.
-            if (model.hasOwnProperty('ep')) {
-                const eps = model.ep(device);
-                endpoint = eps.hasOwnProperty(topic.postfix) ? eps[topic.postfix] : null;
-                if (endpoint === null && eps.hasOwnProperty('default')) {
-                    endpoint = eps['default'];
-                }
-            }
-
-            converters = model.toZigbee;
-            options = model.options || {};
-        } else if (entity.type === 'group') {
-            converters = groupConverters;
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = {};
-        if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            json[topic.attribute] = message.toString();
-        } else {
-            try {
-                json = JSON.parse(message);
-            } catch (e) {
-                // Cannot be parsed to JSON, assume state message.
-                json = {state: message.toString()};
-            }
-        }
-
-        /**
-         * Home Assistant always publishes 'state', even when e.g. only setting
-         * the color temperature. This would lead to 2 zigbee publishes, where the first one
-         * (state) is probably unecessary.
-         */
-        if (settings.get().homeassistant) {
-            const deviceState = this.state.get(entity.ID);
-            const hasColorTemp = json.hasOwnProperty('color_temp');
-            const hasColor = json.hasOwnProperty('color');
-            const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
-            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
-                delete json.state;
-                logger.debug('Skipping state because of Home Assistant');
-            }
-        }
-
-        // Ensure that state and brightness are executed before other commands.
-        const keys = Object.keys(json);
-        keys.sort((a, b) => (['state', 'brightness'].includes(a) ? -1 : 1));
-
-        // For each key in the JSON message find the matching converter.
-        const usedConverters = [];
-        keys.forEach((key) => {
-            const converter = converters.find((c) => c.key.includes(key));
-
-            if (usedConverters.includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
-                return;
-            }
-
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                return;
-            }
-
-            // Converter didn't return a result, skip
-            const convertedResults = converter.convert(key, json[key], json, topic.type, topic.postfix, options);
-            if (!convertedResults || !convertedResults.length) {
-                return;
-            }
-
-            convertedResults.forEach((converted) => {
-                this.zigbee.publish(
-                    entity.ID,
-                    entity.type,
-                    converted.cid,
-                    converted.cmd,
-                    converted.cmdType,
-                    converted.zclData,
-                    converted.cfg,
-                    endpoint,
-                    (error, rsp) => {
-                        if (!error) {
-                            this.handlePublished(entity, topic, converter, converted, key, json[key]);
-                        } else {
-                            this.handlePublishError(entity, message, error);
-                        }
-                    }
-                );
-
-                // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-                // So here we re-read the value after a specified timeout, this timeout could for example be the
-                // transition time of a color change or for forcing a state read for devices that don't
-                // automatically report a new state when set.
-                // When reporting is requested for a device (report: true in device-specific settings) we won't
-                // ever issue a read here, as we assume the device will properly report changes.
-                // Only do this when the retrieve_state option is enabled for this device.
-                const deviceSettings = settings.getDevice(entity.ID);
-                if (topic.type === 'set' && entity.type === 'device' &&
-                    converted.hasOwnProperty('readAfterWriteTime') && deviceSettings && deviceSettings.retrieve_state) {
-                    const getConvertedResults = converter.convert(key, json[key], json, 'get', '', options);
-                    getConvertedResults.forEach((getConverted) => {
-                        setTimeout(() => {
-                            this.zigbee.publish(
-                                entity.ID, entity.type, getConverted.cid, getConverted.cmd, getConverted.cmdType,
-                                getConverted.zclData, getConverted.cfg, endpoint, () => {}
-                            );
-                        }, converted.readAfterWriteTime);
-                    });
-                }
-
-                usedConverters.push(converter);
-            });
-        });
-
-        return true;
-    }
-}
-
-module.exports = DevicePublish;

lib/extension/deviceReceive.js

@@ -1,212 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const debounce = require('debounce');
-
-/**
- * This extensions handles messages received from devices.
- */
-class DeviceReceive {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-        this.coordinator = null;
-        this.elapsed = {};
-        this.debouncers = {};
-    }
-
-    onZigbeeStarted() {
-        this.coordinator = this.zigbee.getCoordinator().device.ieeeAddr;
-    }
-
-    publishDebounce(ieeeAddr, payload, time) {
-        if (!this.debouncers[ieeeAddr]) {
-            this.debouncers[ieeeAddr] = {
-                payload: {},
-                publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload);
-                    this.debouncers[ieeeAddr].payload = {};
-                }, time * 1000),
-            };
-        }
-
-        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
-        this.debouncers[ieeeAddr].publish();
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (message.type == 'devInterview') {
-            if (!settings.getDevice(message.data)) {
-                logger.info('Connecting with device...');
-                this.mqtt.log('pairing', 'connecting with device');
-            }
-
-            return;
-        }
-
-        if (message.type == 'devIncoming') {
-            logger.info('Device incoming...');
-            this.mqtt.log('pairing', 'device incoming');
-        }
-
-        if (!device) {
-            logger.warn('Message without device!');
-            return;
-        }
-
-        if (device.ieeeAddr === this.coordinator) {
-            logger.debug('Ignoring message from coordinator');
-            return;
-        }
-
-        // Check if this is a new device.
-        const settingsDevice = settings.getDevice(device.ieeeAddr);
-        if (!settingsDevice) {
-            logger.info(`New device '${device.modelId}' with address ${device.ieeeAddr} connected!`);
-            settings.addDevice(device.ieeeAddr);
-            this.mqtt.log('device_connected', device.ieeeAddr, {modelID: device.modelId});
-        }
-
-        if (!mappedDevice) {
-            logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-            return;
-        }
-
-        // After this point we cant handle message withoud cid or cmdId anymore.
-        if (!message.data || (!message.data.cid && !message.data.cmdId)) {
-            return;
-        }
-
-        /**
-         * Don't handle re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The source address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the source address of the end device.
-         * As the same message is also received directly from the end device, it makes no sense
-         * to handle these messages.
-         */
-        const hasGroupID = message.hasOwnProperty('groupid') && message.groupid != 0;
-        if (utils.isXiaomiDevice(device) && utils.isRouter(device) && hasGroupID) {
-            logger.debug('Skipping re-transmitted Xiaomi message');
-            return;
-        }
-
-        // Find a conveter for this message.
-        const cid = message.data.cid;
-        const cmdId = message.data.cmdId;
-        const converters = mappedDevice.fromZigbee.filter((c) => {
-            if (cid) {
-                // readRsp messages have the same structure as attReport messages.
-                // search for attReport converters on readRsp.
-                if (c.cid === cid) {
-                    if (c.type instanceof Array) {
-                        return c.type.includes(message.type);
-                    } else {
-                        return c.type === message.type;
-                    }
-                }
-
-                return false;
-            } else if (cmdId) {
-                return c.cmd === cmdId;
-            }
-
-            return false;
-        });
-
-        // Check if there is an available converter
-        if (!converters.length) {
-            if (cid) {
-                // Don't log readRsp messages, they are not interesting most of the time.
-                if (message.type === 'readRsp') {
-                    return;
-                }
-
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cid '${cid}', ` +
-                    `type '${message.type}' and data '${JSON.stringify(message.data)}'`
-                );
-            } else if (cmdId) {
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cmd '${cmdId}' ` +
-                    `and data '${JSON.stringify(message.data)}'`
-                );
-            }
-
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
-            return;
-        }
-
-        // Convert this Zigbee message to a MQTT message.
-        // Get payload for the message.
-        // - If a payload is returned publish it to the MQTT broker
-        // - If NO payload is returned do nothing. This is for non-standard behaviour
-        //   for e.g. click switches where we need to count number of clicks and detect long presses.
-        const publish = (payload) => {
-            // Add device linkquality.
-            if (message.hasOwnProperty('linkquality')) {
-                payload.linkquality = message.linkquality;
-            }
-
-            // Add last seen timestamp
-            const now = Date.now();
-            if (settings.get().advanced.last_seen !== 'disable') {
-                payload.last_seen = utils.formatDate(now, settings.get().advanced.last_seen);
-            }
-
-            if (settings.get().advanced.elapsed) {
-                if (this.elapsed[device.ieeeAddr]) {
-                    payload.elapsed = now - this.elapsed[device.ieeeAddr];
-                }
-
-                this.elapsed[device.ieeeAddr] = now;
-            }
-
-            // Check if we have to debounce
-            if (settingsDevice && settingsDevice.hasOwnProperty('debounce')) {
-                this.publishDebounce(device.ieeeAddr, payload, settingsDevice.debounce);
-            } else {
-                this.publishEntityState(device.ieeeAddr, payload);
-
-                if (settings.get().homeassistant) {
-                    /**
-                     * Publish an empty value for click and action payload, in this way Home Assistant
-                     * can use Home Assistant entities in automations.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
-                     */
-                    Object.keys(payload).forEach((key) => {
-                        if (['action', 'click'].includes(key)) {
-                            const counterPayload = {};
-                            counterPayload[key] = '';
-                            this.publishEntityState(device.ieeeAddr, counterPayload);
-                        }
-                    });
-                }
-            }
-        };
-
-        let payload = {};
-        converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, message, publish, options);
-
-            if (converted) {
-                payload = {...payload, ...converted};
-            }
-        });
-
-        if (Object.keys(payload).length) {
-            publish(payload);
-        }
-    }
-}
-
-module.exports = DeviceReceive;

lib/extension/deviceReport.js

@@ -1,115 +0,0 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-const CC2530Router = zigbeeShepherdConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
-const utils = require('../util/utils');
-
-const candidates = {
-    'genOnOff': {
-        attrs: ['onOff'],
-        reportIntervalMin: 3,
-        reportIntervalMax: 300,
-        reportableChange: 0,
-    },
-    'genLevelCtrl': {
-        attrs: ['currentLevel'],
-    },
-    'lightingColorCtrl': {
-        attrs: ['colorTemperature', 'currentX', 'currentY'],
-    },
-};
-
-const reportInterval = {
-    min: 3,
-    max: 3600,
-};
-
-const reportableChange = 1;
-
-class DeviceReport {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    setupReporting(mappedDevice, device) {
-        let epId = null;
-
-        if (mappedDevice === CC2530Router) {
-            logger.debug('Not setting up reporting for CC2530 router');
-            return;
-        }
-
-        // Check if this device uses a different epId.
-        if (mappedDevice.hasOwnProperty('ep')) {
-            const eps = mappedDevice.ep(device);
-            epId = eps[''] || null;
-        }
-
-        const endpoint = this.zigbee.getEndpoint(device.ieeeAddr, epId);
-
-        if (!endpoint) {
-            logger.error(`Failed to setup reporting for ${device.ieeeAddr}, endpoint not found`);
-            return;
-        }
-
-        logger.debug(`Setting up reporting for ${device.ieeeAddr}`);
-        Object.values(endpoint.clusters).filter((c) => c).forEach((c) => {
-            const cluster = c.attrs.cid;
-            if (candidates[cluster]) {
-                const candidate = candidates[cluster];
-                let attributeNames = candidate.attrs.filter((a) => c.attrs.hasOwnProperty(a));
-
-                // Sometimes a cluster has no attributes, in this case setup reporting for all attributes.
-                attributeNames = attributeNames.length ? attributeNames : candidate.attrs;
-
-                const attributes = attributeNames.map((attribute) => {
-                    return {
-                        attr: attribute,
-                        min: candidate.hasOwnProperty('reportIntervalMin')?
-                            candidate.reportIntervalMin:reportInterval.min,
-                        max: candidate.hasOwnProperty('reportIntervalMax')?
-                            candidate.reportIntervalMax:reportInterval.max,
-                        change: candidate.hasOwnProperty('reportableChange')?
-                            candidate.reportableChange:reportableChange,
-                    };
-                });
-
-                if (attributes.length > 0) {
-                    this.zigbee.report(endpoint, cluster, attributes);
-                }
-            }
-        });
-    }
-
-    shouldSetupReporting(device) {
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients()
-            .filter((d) => this.shouldSetupReporting(d))
-            .forEach((device) => {
-                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-
-                if (mappedDevice) {
-                    this.setupReporting(mappedDevice, device);
-                }
-            });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        // Handle messages of type endDeviceAnnce and devIncoming.
-        // This message is typically send when a device comes online after being powered off
-        // Ikea TRADFRI tend to forget their reporting after powered off.
-        // Re-setup reporting.
-        // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (device && mappedDevice && ['endDeviceAnnce', 'devIncoming'].includes(message.type) &&
-            this.shouldSetupReporting(device)) {
-            this.setupReporting(mappedDevice, device);
-        }
-    }
-}
-
-module.exports = DeviceReport;

lib/extension/extension.js

@@ -1,8 +1,4 @@
-/**
- * This extensions is for documentation purposes only.
- * It describes all methods that are called by the controller.
- */
-class ExtensionTemplate {
+class Extension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -10,31 +6,34 @@ class ExtensionTemplate {
      * @param {MQTT} mqtt MQTT controller
      * @param {State} state State controller
      * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
      */
-    constructor(zigbee, mqtt, state, publishEntityState) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.zigbee = zigbee;
         this.mqtt = mqtt;
         this.state = state;
         this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
     }
 
     /**
      * This method is called by the controller once Zigbee has been started.
      */
-    onZigbeeStarted() {}
+    // onZigbeeStarted() {}
 
     /**
      * This method is called by the controller once connected to the MQTT server.
      */
-    onMQTTConnected() {}
+    // onMQTTConnected() {}
 
     /**
      * Is called when a Zigbee message from a device is received.
-     * @param {Object?} message The received message (can be null)
-     * @param {Object?} device The device of the message (can be null)
-     * @param {Object?} mappedDevice The mapped device (can be null)
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} settingsDevice Device settings
      */
-    onZigbeeMessage(message, device, mappedDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -42,14 +41,12 @@ class ExtensionTemplate {
      * @param {Object} message The received message
      * @return {boolean} if the message was handled
      */
-    onMQTTMessage(topic, message) {
-        return false;
-    }
+    // onMQTTMessage(topic, message) {}
 
     /**
      * Is called once the extension has to stop
      */
-    stop() {}
+    // stop() {}
 }
 
-module.exports = ExtensionTemplate;
+module.exports = Extension;

lib/extension/groups.js

@@ -1,233 +1,265 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const data = require('../util/data');
+const Extension = require('./extension');
 const utils = require('../util/utils');
-const fs = require('fs');
-const diff = require('deep-diff');
+const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/.+/(remove|add|remove_all)$`);
-const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(remove|add|remove_all)$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
+class Groups extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
-
-        this.groupsCacheFile = data.joinPathStorage('groups_cache.json');
-        this.groupsCache = this.readGroupsCache();
-    }
-
-    readGroupsCache() {
-        return fs.existsSync(this.groupsCacheFile) ? JSON.parse(fs.readFileSync(this.groupsCacheFile, 'utf8')) : {};
-    }
-
-    writeGroupsCache() {
-        fs.writeFileSync(this.groupsCacheFile, JSON.stringify(this.groupsCache), 'utf8');
+        this.legacyApi = settings.get().advanced.legacy_api;
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/add`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove_all`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-    }
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
 
-    apply(from, to) {
-        const sortGroups = (obj) => Object.keys(obj).forEach((key) => obj[key] = obj[key].sort());
-
-        sortGroups(from);
-        sortGroups(to);
-
-        const differences = diff(from, to);
-        if (differences) {
-            differences.forEach((diff) => {
-                const groupID = diff.path[0];
-
-                if (diff.kind === 'N') {
-                    diff.rhs.forEach((ID) => this.updateDeviceGroup(ID, 'add', groupID));
-                } else if (diff.kind === 'A') {
-                    if (diff.item.lhs) {
-                        this.updateDeviceGroup(diff.item.lhs, 'remove', groupID);
-                    } else {
-                        this.updateDeviceGroup(diff.item.rhs, 'add', groupID);
-                    }
-                } else if (diff.kind === 'D') {
-                    diff.lhs.forEach((ID) => this.updateDeviceGroup(ID, 'remove', groupID));
-                } else if (diff.kind === 'E') {
-                    this.updateDeviceGroup(diff.rhs, 'add', groupID);
-                    this.updateDeviceGroup(diff.lhs, 'remove', groupID);
-                }
-            });
+            for (let step = 1; step < 20; step++) {
+                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+                this.mqtt.subscribe(`${topic}remove`);
+                this.mqtt.subscribe(`${topic}add`);
+                this.mqtt.subscribe(`${topic}remove_all`);
+            }
         }
     }
 
-    parseID(ID) {
-        let entityID = ID;
-        let endpointID = null;
-        const postfix = utils.getPostfixes().find((p) => entityID.endsWith(`/${p}`));
-        if (postfix) {
-            // Found a postfix, retrieve the endpoint which correspodns to the postfix
-            entityID = entityID.substring(0, entityID.length - (postfix.length + 1));
-            const endpoint = utils.getEndpointByEntityID(this.zigbee, entityID, postfix);
+    async onZigbeeStarted() {
+        await this.syncGroupsWithSettings();
+        this.eventBus.on('stateChange', this.onStateChange);
+    }
 
-            if (!endpoint) {
-                return;
+    async syncGroupsWithSettings() {
+        const settingsGroups = settings.getGroups();
+        const zigbeeGroups = this.zigbee.getGroups();
+        for (const settingGroup of settingsGroups) {
+            const groupID = settingGroup.ID;
+            const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
+            const settingsEntity = settingGroup.devices.map((d) => {
+                const entity = this.zigbee.resolveEntity(d);
+                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
+                return entity;
+            }).filter((e) => e != null);
+
+            // In settings but not in zigbee
+            for (const entity of settingsEntity) {
+                if (!zigbeeGroup.hasMember(entity.endpoint)) {
+                    logger.info(`Adding '${entity.name}' to group '${settingGroup.friendlyName}'`);
+                    await entity.endpoint.addToGroup(zigbeeGroup);
+                }
             }
 
-            endpointID = endpoint.epId;
+            // In zigbee but not in settings
+            for (const endpoint of zigbeeGroup.members) {
+                if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
+                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
+                    logger.info(`Removing '${deviceSettings.friendlyName}' from group '${settingGroup.friendlyName}'`);
+                    await endpoint.removeFromGroup(zigbeeGroup);
+                }
+            }
         }
 
-        return {endpointID, entityID};
+        // eslint-disable-next-line
+        for (const zigbeeGroup of zigbeeGroups) {
+            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
+                for (const endpoint of zigbeeGroup.members) {
+                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
+                    logger.info(`Removing '${deviceSettings.friendlyName}' from group ${zigbeeGroup.groupID}`);
+                    await endpoint.removeFromGroup(zigbeeGroup);
+                }
+            }
+        }
     }
 
-    getGroupsOfDevice(entityID) {
-        return Object.keys(settings.getGroups()).filter((groupID) => {
-            return settings.getGroup(groupID).devices.includes(entityID);
-        });
-    }
+    async onStateChange(data) {
+        const reason = 'group_optimistic';
+        if (data.reason === reason) {
+            return;
+        }
 
-    onStateChange(ieeeAddr, from, to) {
         const properties = ['state', 'brightness', 'color_temp', 'color'];
         const payload = {};
 
         properties.forEach((prop) => {
-            if (to.hasOwnProperty(prop) && (!from || from[prop] != to[prop])) {
-                payload[prop] = to[prop];
+            if (data.to.hasOwnProperty(prop)) {
+                payload[prop] = data.to[prop];
             }
         });
 
-        if (Object.keys(payload)) {
-            const groups = this.getGroupsOfDevice(ieeeAddr);
-            groups.forEach((groupID) => {
-                this.publishEntityState(groupID, payload);
+        if (Object.keys(payload).length) {
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
+            const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
+                const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
+                return settingsGroup && settingsGroup.optimistic;
             });
-        }
-    }
 
-    onZigbeeStarted() {
-        this.state.registerOnStateChangeListener(this.onStateChange);
-
-        const settingsGroups = {};
-        Object.keys(settings.getGroups()).forEach((groupID) => {
-            settingsGroups[groupID] = settings.getGroup(groupID).devices;
-        });
-
-        this.apply(this.groupsCache, settingsGroups);
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex) && !topic.match(topicRegexRemoveAll)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/group/`, '');
-
-        // Parse type from topic
-        const type = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-        // Remove type from topic
-        topic = topic.replace(`/${type}`, '');
-
-        return {friendly_name: type === 'remove_all' ? null : topic, type};
-    }
-
-    updateDeviceGroup(ID, cmd, groupID) {
-        let payload = null;
-        const orignalCmd = cmd;
-        if (cmd === 'add') {
-            payload = {groupid: groupID, groupname: ''};
-            cmd = 'add';
-        } else if (cmd === 'remove') {
-            payload = {groupid: groupID};
-            cmd = 'remove';
-        } else if (cmd === 'remove_all') {
-            payload = {};
-            cmd = 'removeAll';
-        }
-
-        const {entityID, endpointID} = this.parseID(ID);
-        const ieeeAddr = settings.resolveEntity(entityID).ID;
-
-        const cb = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to ${cmd} ${ieeeAddr} from ${groupID}`);
+            if (resolvedEntity.type === 'device') {
+                for (const zigbeeGroup of zigbeeGroups) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
+                        }
+                    }
+                }
             } else {
-                logger.info(`Successfully ${cmd} ${ieeeAddr} to ${groupID}`);
-
-                // Log to MQTT
-                this.mqtt.log({
-                    device: settings.getDevice(ieeeAddr).friendly_name,
-                    group: groupID,
-                    action: orignalCmd,
-                });
-
-                // Update group cache
-                if (cmd === 'add') {
-                    if (!this.groupsCache[groupID]) {
-                        this.groupsCache[groupID] = [];
+                const groupIDsToPublish = new Set();
+                for (const member of resolvedEntity.group.members) {
+                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
+                    for (const zigbeeGroup of zigbeeGroups) {
+                        if (zigbeeGroup.hasMember(member)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                                groupIDsToPublish.add(zigbeeGroup.groupID);
+                            }
+                        }
                     }
-
-                    if (!this.groupsCache[groupID].includes(ieeeAddr)) {
-                        this.groupsCache[groupID].push(ieeeAddr);
-                    }
-                } else if (cmd === 'remove') {
-                    if (this.groupsCache[groupID]) {
-                        this.groupsCache[groupID] = this.groupsCache[groupID].filter((device) => device != ieeeAddr);
-                    }
-                } else if (cmd === 'removeAll') {
-                    Object.keys(this.groupsCache).forEach((groupID_) => {
-                        this.groupsCache[groupID_] = this.groupsCache[groupID_].filter((device) => device != ieeeAddr);
-                    });
                 }
-
-                this.writeGroupsCache();
-
-                // Update settings
-                if (cmd === 'add') {
-                    settings.addDeviceToGroup(groupID, ieeeAddr);
-                } else if (cmd === 'remove') {
-                    settings.removeDeviceFromGroup(groupID, ieeeAddr);
-                } else if (cmd === 'removeAll') {
-                    Object.keys(settings.get().groups).forEach((groupID_) => {
-                        settings.removeDeviceFromGroup(groupID_, ieeeAddr);
-                    });
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
+                for (const groupID of groupIDsToPublish) {
+                    await this.publishEntityState(groupID, payload, reason);
                 }
             }
-        };
-
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', cmd, 'functional',
-            payload, null, endpointID, cb,
-        );
+        }
     }
 
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
+    areAllMembersOff(zigbeeGroup) {
+        for (const member of zigbeeGroup.members) {
+            const device = member.getDevice();
+            if (this.state.exists(device.ieeeAddr)) {
+                const state = this.state.get(device.ieeeAddr);
+                if (state && state.state === 'ON') {
+                    return false;
+                }
+            }
         }
+        return true;
+    }
 
-        // Find ID of this group.
-        let groupID = null;
-        if (topic.type !== 'remove_all') {
-            groupID = settings.getGroupIDByFriendlyName(topic.friendly_name);
-            if (!groupID) {
-                logger.error(`Group with friendly_name '${topic.friendly_name}' doesn't exist`);
-                return;
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
+
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            const topicMatch = topic.match(legacyTopicRegex);
+            if (topicMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
+                type = topicMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else if (topic.match(legacyTopicRegexRemoveAll)) {
+                type = 'remove_all';
+            } else {
+                return {};
             }
 
-            groupID = groupID.toString();
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
         }
 
-        // Send command to the device.
-        this.updateDeviceGroup(message.toString(), topic.type, groupID);
+        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+    }
 
-        return true;
+    async onMQTTMessage(topic, message) {
+        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
+            this.parseMQTTMessage(topic, message);
+        if (!type) return;
+
+        const keys = [
+            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+        ];
+
+        const definition = resolvedEntityDevice.definition;
+        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
+        const endpointName = endpoints ?
+            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+
+        if (endpointName) {
+            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+        }
+
+        if (!hasEndpointName) {
+            keys.push(resolvedEntityDevice.name);
+            keys.push(resolvedEntityDevice.device.ieeeAddr);
+        }
+
+        if (type === 'add') {
+            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_add`, message: payload}),
+                );
+            }
+        } else if (type === 'remove') {
+            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_remove`, message: payload}),
+                );
+            }
+        } else { // remove_all
+            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+            await resolvedEntityDevice.endpoint.removeFromAllGroups();
+            for (const settingsGroup of settings.getGroups()) {
+                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: resolvedEntityDevice.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                    );
+                }
+            }
+        }
     }
 }
 

lib/extension/legacy/bridgeLegacy.js

@@ -0,0 +1,463 @@
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const utils = require('../../util/utils');
+const assert = require('assert');
+const Extension = require('../extension');
+
+const configRegex =
+    new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
+const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
+
+class BridgeLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        // Bind functions
+        this.permitJoin = this.permitJoin.bind(this);
+        this.lastSeen = this.lastSeen.bind(this);
+        this.elapsed = this.elapsed.bind(this);
+        this.reset = this.reset.bind(this);
+        this.logLevel = this.logLevel.bind(this);
+        this.devices = this.devices.bind(this);
+        this.groups = this.groups.bind(this);
+        this.rename = this.rename.bind(this);
+        this.renameLast = this.renameLast.bind(this);
+        this.remove = this.remove.bind(this);
+        this.forceRemove = this.forceRemove.bind(this);
+        this.ban = this.ban.bind(this);
+        this.deviceOptions = this.deviceOptions.bind(this);
+        this.addGroup = this.addGroup.bind(this);
+        this.removeGroup = this.removeGroup.bind(this);
+        this.whitelist = this.whitelist.bind(this);
+        this.touchlinkFactoryReset = this.touchlinkFactoryReset.bind(this);
+
+        this.lastJoinedDeviceName = null;
+
+        // Set supported options
+        this.supportedOptions = {
+            'permit_join': this.permitJoin,
+            'last_seen': this.lastSeen,
+            'elapsed': this.elapsed,
+            'reset': this.reset,
+            'log_level': this.logLevel,
+            'devices': this.devices,
+            'groups': this.groups,
+            'devices/get': this.devices,
+            'rename': this.rename,
+            'rename_last': this.renameLast,
+            'remove': this.remove,
+            'force_remove': this.forceRemove,
+            'ban': this.ban,
+            'device_options': this.deviceOptions,
+            'add_group': this.addGroup,
+            'remove_group': this.removeGroup,
+            'whitelist': this.whitelist,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+        };
+    }
+
+    whitelist(topic, message) {
+        try {
+            const entity = settings.getEntity(message);
+            assert(entity, `Entity '${message}' does not exist`);
+            settings.whitelistDevice(entity.ID);
+            logger.info(`Whitelisted '${entity.friendlyName}'`);
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to whitelist '${message}' '${error}'`);
+        }
+    }
+
+    deviceOptions(topic, message) {
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error('Failed to parse message as JSON');
+            return;
+        }
+
+        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
+            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
+            return;
+        }
+
+        const entity = settings.getEntity(json.friendly_name);
+        assert(entity, `Entity '${json.friendly_name}' does not exist`);
+        settings.changeDeviceOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
+    }
+
+    async permitJoin(topic, message) {
+        await this.zigbee.permitJoin(message.toLowerCase() === 'true');
+        this.publish();
+    }
+
+    async reset(topic, message) {
+        try {
+            await this.zigbee.reset('soft');
+            logger.info('Soft resetted ZNP');
+        } catch (error) {
+            logger.error('Soft reset failed');
+        }
+    }
+
+    lastSeen(topic, message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        logger.info(`Set last_seen to ${message}`);
+    }
+
+    elapsed(topic, message) {
+        const allowed = ['true', 'false'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'elapsed'], message === 'true');
+        logger.info(`Set elapsed to ${message}`);
+    }
+
+    logLevel(topic, message) {
+        const level = message.toLowerCase();
+        if (allowedLogLevels.includes(level)) {
+            logger.info(`Switching log level to '${level}'`);
+            logger.setLevel(level);
+        } else {
+            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
+        }
+
+        this.publish();
+    }
+
+    async devices(topic, message) {
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const devices = this.zigbee.getDevices().map((device) => {
+            const payload = {
+                ieeeAddr: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+            };
+
+            if (device.type !== 'Coordinator') {
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const friendlyDevice = settings.getDevice(device.ieeeAddr);
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
+                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+                payload.manufacturerID = device.manufacturerID;
+                payload.manufacturerName = device.manufacturerName;
+                payload.powerSource = device.powerSource;
+                payload.modelID = device.modelID;
+                payload.hardwareVersion = device.hardwareVersion;
+                payload.softwareBuildID = device.softwareBuildID;
+                payload.dateCode = device.dateCode;
+                payload.lastSeen = device.lastSeen;
+            } else {
+                payload.friendly_name = 'Coordinator';
+                payload.softwareBuildID = coordinator.type;
+                payload.dateCode = coordinator.meta.revision.toString();
+                payload.lastSeen = Date.now();
+            }
+
+            return payload;
+        });
+
+        if (topic.split('/').pop() == 'get') {
+            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
+        } else {
+            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
+        }
+    }
+
+    groups(topic, message) {
+        const payload = settings.getGroups().map((g) => {
+            const group = {...g};
+            delete group.friendlyName;
+            return group;
+        });
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
+    }
+
+    rename(topic, message) {
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
+
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error(invalid);
+            return;
+        }
+
+        // Validate message
+        if (!json.new || !json.old) {
+            logger.error(invalid);
+            return;
+        }
+
+        this._renameInternal(json.old, json.new);
+    }
+
+    renameLast(topic, message) {
+        if (!this.lastJoinedDeviceName) {
+            logger.error(`Cannot rename last joined device, no device has joined during this session`);
+            return;
+        }
+
+        this._renameInternal(this.lastJoinedDeviceName, message);
+    }
+
+    _renameInternal(from, to) {
+        try {
+            const isGroup = settings.getGroup(from) !== null;
+            settings.changeFriendlyName(from, to);
+            logger.info(`Successfully renamed - ${from} to ${to} `);
+            const entity = this.zigbee.resolveEntity(to);
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to rename - ${from} to ${to}`);
+        }
+    }
+
+    addGroup(topic, message) {
+        let id = null;
+        let name = null;
+        try {
+            // json payload with id and friendly_name
+            const json = JSON.parse(message);
+            if (json.hasOwnProperty('id')) {
+                id = json.id;
+                name = `group_${id}`;
+            }
+            if (json.hasOwnProperty('friendly_name')) {
+                name = json.friendly_name;
+            }
+        } catch (e) {
+            // just friendly_name
+            name = message;
+        }
+
+        if (name == null) {
+            logger.error('Failed to add group, missing friendly_name!');
+            return;
+        }
+
+        const group = settings.addGroup(name, id);
+        this.zigbee.createGroup(group.ID);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
+        logger.info(`Added group '${name}'`);
+    }
+
+    removeGroup(topic, message) {
+        const name = message;
+        const entity = this.zigbee.resolveEntity(message);
+        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+        settings.removeGroup(message);
+        entity.group.removeFromDatabase();
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
+        logger.info(`Removed group '${name}'`);
+    }
+
+    async forceRemove(topic, message) {
+        await this.removeForceRemoveOrBan('force_remove', message);
+    }
+
+    async remove(topic, message) {
+        await this.removeForceRemoveOrBan('remove', message);
+    }
+
+    async ban(topic, message) {
+        await this.removeForceRemoveOrBan('ban', message);
+    }
+
+    async removeForceRemoveOrBan(action, message) {
+        const entity = this.zigbee.resolveEntity(message.trim());
+        const lookup = {
+            ban: ['banned', 'Banning', 'ban'],
+            force_remove: ['force_removed', 'Force removing', 'force remove'],
+            remove: ['removed', 'Removing', 'remove'],
+        };
+
+        if (!entity) {
+            logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            return;
+        }
+
+        const cleanup = () => {
+            // Fire event
+            this.eventBus.emit('deviceRemoved', {device: entity.device});
+
+            // Remove from configuration.yaml
+            settings.removeDevice(entity.settings.ID);
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
+        };
+
+        try {
+            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            if (action === 'force_remove') {
+                await entity.device.removeFromDatabase();
+            } else {
+                await entity.device.removeFromNetwork();
+            }
+
+            cleanup();
+        } catch (error) {
+            logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+            // eslint-disable-next-line
+            logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+        }
+
+        if (action === 'ban') {
+            settings.banDevice(entity.settings.ID);
+        }
+    }
+
+    async onMQTTConnected() {
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
+        await this.publish();
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (!topic.match(configRegex)) {
+            return false;
+        }
+
+        const option = topic.match(configRegex)[1];
+
+        if (!this.supportedOptions.hasOwnProperty(option)) {
+            return false;
+        }
+
+        await this.supportedOptions[option](topic, message);
+
+        return true;
+    }
+
+    async publish() {
+        const info = await utils.getZigbee2mqttVersion();
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const topic = `bridge/config`;
+        const payload = {
+            version: info.version,
+            commit: info.commitHash,
+            coordinator,
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
+        if (type === 'deviceJoined') {
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        logger.info('Starting touchlink factory reset...');
+        this.mqtt.publish(
+            'bridge/log',
+            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
+        const result = await this.zigbee.touchlinkFactoryReset();
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
+        } else {
+            logger.warn('Failed to factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
+        }
+    }
+}
+
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -0,0 +1,46 @@
+/* istanbul ignore file */
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
+
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
+
+class DeviceGroupMembership extends Extension {
+    onMQTTConnected() {
+        for (let step = 1; step < 20; step++) {
+            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
+            this.mqtt.subscribe(topic);
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const entity = this.zigbee.resolveEntity(match[1]);
+        if (!entity || entity.type !== 'device') {
+            logger.error(`Device '${match[1]}' does not exist`);
+            return;
+        }
+
+        const response = await entity.endpoint.command(
+            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
+        );
+
+        const {grouplist, capacity} = response;
+        const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
+        let msgCapacity;
+        if (capacity === 254) {
+            msgCapacity = 'it can be a part of at least 1 more group';
+        } else {
+            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
+        }
+        logger.info(`${msgGroupList} and ${msgCapacity}`);
+
+        this.publishEntityState(entity.device.ieeeAddr, {group_list: grouplist, group_capacity: capacity});
+    }
+}
+
+module.exports = DeviceGroupMembership;

lib/extension/legacy/softReset.js

@@ -0,0 +1,66 @@
+/* istanbul ignore file */
+// DEPRECATED
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
+
+/**
+ * This extensions soft resets the ZNP after a certain timeout.
+ */
+class SoftReset extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.timer = null;
+        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+    }
+
+    onZigbeeStarted() {
+        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        this.resetTimer();
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    resetTimer() {
+        if (this.timeout === 0) {
+            return;
+        }
+
+        this.clearTimer();
+        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
+    }
+
+    async handleTimeout() {
+        logger.warn('Soft reset timeout triggered');
+
+        try {
+            await this.zigbee.reset('soft');
+            logger.warn('Soft resetted ZNP due to timeout');
+        } catch (error) {
+            logger.warn('Soft reset failed, trying stop/start');
+
+            await this.zigbee.stop();
+            logger.warn('Zigbee stopped');
+
+            try {
+                await this.zigbee.start();
+            } catch (error) {
+                logger.error('Failed to restart!');
+            }
+        }
+
+        this.resetTimer();
+    }
+
+    onZigbeeEvent() {
+        this.resetTimer();
+    }
+}
+
+module.exports = SoftReset;

lib/extension/livolo.js

@@ -1,183 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(1);
-const logger = require('../util/logger');
-
-const foundationCfg = {manufSpec: 0, disDefaultRsp: 0};
-
-/**
- * Extension required for Livolo device support.
- */
-class Livolo {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-        this.configured = {};
-    }
-
-    onZigbeeStarted() {
-        this.startTimer();
-    }
-
-    _resetDeviceState(ieeeAddr) {
-        this.configured[ieeeAddr] = {
-            stage: 0,
-            retry: 0,
-            waitresp: false,
-        };
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (!device) {
-            return;
-        }
-
-        if ((message.type == 'devInterview') ||
-            (message.type == 'devIncoming') ||
-            (message.type == 'endDeviceAnnce') ||
-            (message.type == 'devStatus')) {
-            if (this.configured.hasOwnProperty(device.ieeeAddr)) {
-                logger.info(`LIVOLO ${device.ieeeAddr}. (Re)joins in the network (after power off?)`);
-                this._resetDeviceState(device.ieeeAddr);
-            }
-            return;
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    _handleCommandRespSimple(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            if (this.ctype === 'toggle') {
-                logger.debug(`LIVOLO ${this.ieeeAddr}. Toggle command error:`, err.message);
-            }
-        } else {
-            logger.debug(`LIVOLO ${this.ieeeAddr}. Sucessfully configured`, rsp);
-            this.ext.configured[this.ieeeAddr].stage = 1; // sucessfully send command
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-        }
-    }
-
-    _handleCommandRespWithData(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            logger.info(`LIVOLO ${this.ieeeAddr}. ${this.cid}.${this.ctype} response error:`, err.message);
-            if (this.ext.configured[this.ieeeAddr].retry >= 3) {
-                // errors in three sequental reads, stop polling, wait for a device message
-                this.device.status = 'offline';
-                logger.info(`LIVOLO ${this.ieeeAddr}. Stopped polling after 3 unsuccessful attempts`);
-            }
-        } else {
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-            if (this.ext.zigbee) {
-                this.ext.zigbee.shepherd.emit('ind:reported', this.ep, this.cid, rsp, this.ep.last_af_msg);
-            }
-        }
-    }
-
-    // msg: { groupid, clusterid, srcaddr, srcendpoint, dstendpoint, wasbroadcast,
-    //   linkquality, securityuse, timestamp, transseqnumber, len, data }
-    _handleAfMessage(msg, ep) {
-        ep.linkquality = msg.linkquality;
-        ep.last_af_msg = msg;
-    }
-
-    _sendToggle(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            const cfg = {};
-            logger.debug(`LIVOLO ${ieeeAddr}. Sending the 'toggle' command. Retry: ${retry}`);
-            ep.functional('genOnOff', 'toggle', [cfg], foundationCfg,
-                this._handleCommandRespSimple.bind({
-                    device: zdev,
-                    ieeeAddr,
-                    cid: 'genOnOff',
-                    ctype: 'toggle',
-                    ext: this,
-                }));
-
-            queueCallback();
-        });
-    }
-
-    _sendPoll(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            ep.foundation('genOnOff', 'read', [{
-                attrId: 0, // onOff
-            }], this._handleCommandRespWithData.bind({
-                device: zdev,
-                ieeeAddr,
-                ep,
-                cid: 'genOnOff',
-                ctype: 'read',
-                ext: this,
-            }));
-
-            queueCallback();
-        });
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => d.manufName && d.manufName.startsWith('LIVOLO')) // LIVOLO
-            .filter((d) => d.type === 'EndDevice') // Filter end devices
-            .filter((d) => d.powerSource && d.powerSource !== 'Battery') // Remove battery powered devices
-            .forEach((d) => {
-                const zdev = this.zigbee.shepherd._findDevByAddr(d.ieeeAddr);
-                if (zdev && zdev.endpoints) {
-                    const eplist = Object.keys(zdev.endpoints).filter((epId) => {
-                        const ep2 = zdev.getEndpoint(epId);
-                        const clist = ep2.getClusterList();
-                        return clist && clist.includes(6); // 6 - genOnOff
-                    });
-
-                    if (eplist.length > 0) {
-                        const ep = zdev.getEndpoint(eplist[0]);
-
-                        ep.onAfIncomingMsg = this._handleAfMessage;
-
-                        if (!this.configured.hasOwnProperty(d.ieeeAddr)) {
-                            this._resetDeviceState(d.ieeeAddr);
-                        }
-
-                        const state = this.configured[d.ieeeAddr];
-                        if (state.waitresp) {
-                            return;
-                        }
-
-                        if (state.retry < 3) {
-                            if (state.stage === 0) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendToggle(zdev, d.ieeeAddr, ep, state.retry);
-                            } else if (state.stage === 1) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendPoll(zdev, d.ieeeAddr, ep, state.retry);
-                            }
-                        }
-                    }
-                }
-            });
-        return true;
-    }
-}
-
-module.exports = Livolo;

lib/extension/networkMap.js

@@ -1,16 +1,18 @@
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+const logger = require('../util/logger');
+const Extension = require('./extension');
 
-class NetworkMap {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.lastSeenMap = {};
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
 
-        // Subscribe to topic.
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
 
         // Bind
         this.raw = this.raw.bind(this);
@@ -23,120 +25,181 @@ class NetworkMap {
         };
     }
 
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device) {
-            this.lastSeenMap[device.ieeeAddr] = Date.now();
-        }
-    }
-
     onMQTTConnected() {
-        this.mqtt.subscribe(this.topic);
-    }
-
-    onMQTTMessage(topic, message) {
-        message = message.toString();
-
-        if (topic === this.topic && this.supportedFormats.hasOwnProperty(message)) {
-            this.zigbee.networkScan((result)=> {
-                const converted = this.supportedFormats[message](this.zigbee, result);
-                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
-            });
-
-            return true;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(this.legacyTopic);
+            this.mqtt.subscribe(this.legacyTopicRoutes);
         }
-
-        return false;
     }
 
-    raw(zigbee, topology) {
+    async onMQTTMessage(topic, message) {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                const converted = this.supportedFormats[message](topology);
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
+        }
+    }
+
+    raw(topology) {
         return JSON.stringify(topology);
     }
 
-    graphviz(zigbee, topology) {
+    graphviz(topology) {
         const colors = settings.get().map_options.graphviz.colors;
 
         let text = 'digraph G {\nnode[shape=record];\n';
-        let devStyle = '';
+        let style = '';
 
-        topology.nodes.forEach((device) => {
+        topology.nodes.forEach((node) => {
             const labels = [];
 
             // Add friendly name
-            labels.push(`${device.friendlyName}`);
+            labels.push(`${node.friendlyName}`);
 
-            // Add the device short network address and scan note (if any)
-            let scanNote = '';
-            if (device.scanfailed.includes('lqi')) {
-                scanNote += ' no lqi';
-            }
-            if (device.scanfailed.includes('rtg')) {
-                scanNote += ' no routes';
-            }
-            labels.push(`0x${device.nwkAddr.toString(16)} ${scanNote}`);
+            // Add the device short network address, ieeaddr and scan note (if any)
+            labels.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
+            );
 
             // Add the device model
-            const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (mappedModel) {
-                labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
-            } else {
-                // This model is not supported by zigbee-shepherd-converters, add zigbee model information, if available
-                const zigbeeModel = [device.manufName, device.modelId].filter((a) => a).join(' ');
-                labels.push(zigbeeModel ? zigbeeModel : 'No model information available');
-            }
-
-            // Add the device status (online/offline) and last_seen timestamp
-            let lastSeen = 'unknown';
-            if (device.type == 'Coordinator') {
-                lastSeen = utils.formatDate(Date.now(), settings.get().advanced.last_seen,
-                    utils.toLocalISOString(new Date(Date.now())));
-            } else {
-                if (this.lastSeenMap[device.ieeeAddr]) {
-                    const lastSeenAgo =
-                        `${new Date(Date.now() - this.lastSeenMap[device.ieeeAddr]).toISOString().substr(11, 8)}s ago`;
-                    lastSeen = utils.formatDate(this.lastSeenMap[device.ieeeAddr],
-                        settings.get().advanced.last_seen, lastSeenAgo);
+            if (node.type !== 'Coordinator') {
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
                 }
             }
-            labels.push(`${device.status} (${lastSeen})`);
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+            }
+
+            labels.push(lastSeen);
 
             // Shape the record according to device type
-            if (device.type == 'Coordinator') {
-                devStyle = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
+            if (node.type == 'Coordinator') {
+                style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
                     `fontcolor="${colors.font.coordinator}"`;
-            } else if (device.type == 'Router') {
-                devStyle = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
+            } else if (node.type == 'Router') {
+                style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
                     `fontcolor="${colors.font.router}"`;
             } else {
-                devStyle = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", `
-                    + `fontcolor="${colors.font.enddevice}"`;
+                style = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", ` +
+                    `fontcolor="${colors.font.enddevice}"`;
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+devStyle+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
              * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.links.filter((e) => (e.sourceIeeeAddr === device.ieeeAddr) || (e.SourceNwkAddr === device.nwkAddr))
-                .forEach((e) => {
-                    const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, '
-                        : (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
-                    const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", `
-                        : `weight=1, color="${colors.line.active}", `;
-                    const textRoutes = e.routes.map((r) => `0x${r.toString(16)}`);
-                    const lineLabels = (!e.routes.length) ? `label="${e.lqi}"`
-                        : `label="${e.lqi}\\n[${textRoutes.join(']\\n[')}]"`;
-                    text += `  "${device.ieeeAddr}" -> "${e.targetIeeeAddr}"`;
-                    text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
-                });
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
+                    (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
+                const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
+                    `weight=1, color="${colors.line.active}", `;
+                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
+                    `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
+            });
         });
 
         text += '}';
 
         return text.replace(/\0/g, '');
     }
+
+    async networkScan(includeRoutes) {
+        logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
+        const lqis = new Map();
+        const routingTables = new Map();
+        const failed = new Map();
+
+        for (const device of devices.filter((d) => d.type != 'EndDevice')) {
+            failed.set(device, []);
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            try {
+                const result = await device.lqi();
+                lqis.set(device, result);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
+            } catch (error) {
+                failed.get(device).push('lqi');
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
+            }
+
+            if (includeRoutes) {
+                try {
+                    const result = await device.routingTable();
+                    routingTables.set(device, result);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
+                } catch (error) {
+                    failed.get(device).push('routingTable');
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
+                }
+            }
+        }
+
+        logger.info(`Network scan finished`);
+
+        const networkMap = {nodes: [], links: []};
+        // Add nodes
+        for (const device of devices) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            networkMap.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
+                networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
+                modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+            });
+        }
+
+        // Add links
+        lqis.forEach((lqi, device) => {
+            for (const neighbor of lqi.neighbors) {
+                if (neighbor.relationship > 3) {
+                    // Relationship is not active, skip it
+                    continue;
+                }
+
+                const link = {
+                    source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
+                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
+                    linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
+                    // DEPRECATED:
+                    sourceIeeeAddr: neighbor.ieeeAddr, targetIeeeAddr: device.ieeeAddr,
+                    sourceNwkAddr: neighbor.networkAddress, lqi: neighbor.linkquality,
+                    relationship: neighbor.relationship,
+                };
+
+                const routingTable = routingTables.get(device);
+                if (routingTable) {
+                    link.routes = routingTable.table
+                        .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
+                }
+
+                networkMap.links.push(link);
+            }
+        });
+
+        return networkMap;
+    }
 }
 
 module.exports = NetworkMap;

lib/extension/onEvent.js

@@ -0,0 +1,37 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -0,0 +1,220 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const assert = require('assert');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const Extension = require('./extension');
+const MINUTES_10 = 1000 * 60 * 10;
+
+class OTAUpdate extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.inProgress = new Set();
+        this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
+    }
+
+    onMQTTConnected() {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        if (supportsOTA) {
+            // When a device does a next image request, it will usually do it a few times after each other
+            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
+            // each time.
+            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+
+            this.lastChecked[data.device.ieeeAddr] = Date.now();
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            this.publishEntityState(data.device.ieeeAddr, {update_available: available});
+
+            if (available) {
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        }
+
+        // Respond to the OTA request:
+        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
+        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
+    }
+
+    async readSoftwareBuildIDAndDateCode(device, update) {
+        try {
+            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
+
+            if (update) {
+                device.softwareBuildID = result.swBuildId;
+                device.dateCode = result.dateCode;
+                device.save();
+            }
+
+            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
+        } catch (e) {
+            return null;
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        let resolvedEntity = null;
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if (!topic.match(legacyTopicRegex)) {
+                return null;
+            }
+
+            resolvedEntity = this.zigbee.resolveEntity(message);
+        } else {
+            return null;
+        }
+
+        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
+        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
+            logger.error(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            return;
+        }
+
+        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
+            return;
+        }
+        this.inProgress.add(resolvedEntity.device.ieeeAddr);
+
+        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
+        if (type === 'check') {
+            const message = `Checking if update available for '${resolvedEntity.name}'`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
+                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
+                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+            } catch (error) {
+                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `check_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        } else { // type === 'update'
+            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const onProgress = (progress, remaining) => {
+                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                    if (remaining) {
+                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                    }
+
+                    logger.info(message);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    }
+                };
+
+                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                const message = `Finished update of '${resolvedEntity.name}'` +
+                    (to ? `, from '${fromS}' to '${toS}'` : ``);
+                logger.info(message);
+                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            } catch (error) {
+                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_failed`, device: resolvedEntity.name};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            }
+        }
+
+        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+    }
+}
+
+module.exports = OTAUpdate;

lib/extension/publish.js

@@ -0,0 +1,256 @@
+
+const settings = require('../util/settings');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const assert = require('assert');
+const Extension = require('./extension');
+
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+
+const groupConverters = [
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
+    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
+    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
+    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+];
+
+class EntityPublish extends Extension {
+    onMQTTConnected() {
+        // Subscribe to topics.
+        const baseTopic = settings.get().mqtt.base_topic;
+        for (let step = 1; step < 20; step++) {
+            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
+            this.mqtt.subscribe(`${topic}set`);
+            this.mqtt.subscribe(`${topic}set/+`);
+            this.mqtt.subscribe(`${topic}get`);
+            this.mqtt.subscribe(`${topic}get/+`);
+        }
+    }
+
+    parseTopic(topic) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
+    }
+
+    async onMQTTMessage(topic, message) {
+        topic = this.parseTopic(topic);
+        if (!topic) {
+            return false;
+        }
+
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
+
+        if (!resolvedEntity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityKey};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityKey}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        let converters = null;
+        let target = null;
+        let options = {};
+        let device = null;
+        let definition = null;
+
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
+                return;
+            }
+
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = resolvedEntity.settings;
+        } else {
+            converters = groupConverters;
+            target = resolvedEntity.group;
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        let json = {};
+        if (topic.hasOwnProperty('attribute') && topic.attribute) {
+            json[topic.attribute] = message;
+        } else {
+            try {
+                json = JSON.parse(message);
+            } catch (e) {
+                // Cannot be parsed to JSON, assume state message.
+                json = {state: message};
+            }
+        }
+
+
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
+        if (settings.get().homeassistant) {
+            const hasColorTemp = json.hasOwnProperty('color_temp');
+            const hasColor = json.hasOwnProperty('color');
+            const hasBrightness = json.hasOwnProperty('brightness');
+            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete json.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
+        const entries = Object.entries(json);
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters = {};
+        for (let [key, value] of entries) {
+            let endpointName = topic.endpointName;
+            let actualTarget = target;
+
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
+                const underscoreIndex = key.lastIndexOf('_');
+                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possibleEndpointName)) {
+                    endpointName = possibleEndpointName;
+                    key = key.substring(0, underscoreIndex);
+                    const device = target.getDevice();
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
+                }
+            }
+
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${json[key]})`);
+                continue;
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {
+                endpoint_name: endpointName,
+                options,
+                message: json,
+                logger,
+                device,
+                state: deviceState,
+                mapped: definition,
+            };
+
+            try {
+                if (topic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
+                    const result = await converter.convertSet(actualTarget, key, value, meta);
+                    if (result && result.state) {
+                        const msg = result.state;
+
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                                if (msg.hasOwnProperty(key)) {
+                                    msg[`${key}_${endpointName}`] = msg[key];
+                                    delete msg[key];
+                                }
+                            }
+                        }
+
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                    }
+
+                    // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+                    // So here we re-read the value after a specified timeout, this timeout could for example be the
+                    // transition time of a color change or for forcing a state read for devices that don't
+                    // automatically report a new state when set.
+                    // When reporting is requested for a device (report: true in device-specific settings) we won't
+                    // ever issue a read here, as we assume the device will properly report changes.
+                    // Only do this when the retrieve_state option is enabled for this device.
+                    if (
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
+                    ) {
+                        setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
+                    }
+                } else if (topic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
+                    await converter.convertGet(actualTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
+            }
+
+            usedConverters[endpointOrGroupID].push(converter);
+        }
+
+        return true;
+    }
+}
+
+module.exports = EntityPublish;

lib/extension/receive.js

@@ -0,0 +1,180 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const debounce = require('debounce');
+const Extension = require('./extension');
+
+class Receive extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.elapsed = {};
+        this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+    }
+
+    async onZigbeeStarted() {
+        this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+    }
+
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
+    publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
+        if (!this.debouncers[ieeeAddr]) {
+            this.debouncers[ieeeAddr] = {
+                payload: {},
+                publish: debounce(() => {
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
+                    this.debouncers[ieeeAddr].payload = {};
+                }, time * 1000),
+            };
+        }
+
+        if (this.isPayloadConflicted(payload, this.debouncers[ieeeAddr].payload, debounceIgnore)) {
+            // publish previous payload immediately
+            this.debouncers[ieeeAddr].publish.flush();
+        }
+
+        // extend debounced payload with current
+        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
+        this.debouncers[ieeeAddr].publish();
+    }
+
+    // if debounce_ignore are specified (Array of strings)
+    // then all newPayload values with key present in debounce_ignore
+    // should equal or be undefined in oldPayload
+    // otherwise payload is conflicted
+    isPayloadConflicted(newPayload, oldPayload, debounceIgnore) {
+        let result = false;
+        Object.keys(oldPayload)
+            .filter((key) => (debounceIgnore || []).includes(key))
+            .forEach((key) => {
+                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
+                    result = true;
+                }
+            });
+
+        return result;
+    }
+
+    shouldProcess(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
+            return false;
+        }
+
+        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
+            logger.debug('Ignoring message from coordinator');
+            return false;
+        }
+
+        /**
+         * Don't handle re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The source address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the source address of the end device.
+         * As the same message is also received directly from the end device, it makes no sense
+         * to handle these messages.
+         */
+        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
+        if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
+            logger.debug('Skipping re-transmitted Xiaomi message');
+            return false;
+        }
+
+        if (!data.device.modelID && data.device.interviewing) {
+            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
+            return false;
+        }
+
+        if (!resolvedEntity.definition) {
+            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            return false;
+        }
+
+        return true;
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
+            return;
+        }
+
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
+            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
+            return c.cluster === data.cluster && type;
+        });
+
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
+            logger.debug(
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
+                `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
+            );
+            return;
+        }
+
+        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
+        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
+
+        // Convert this Zigbee message to a MQTT message.
+        // Get payload for the message.
+        // - If a payload is returned publish it to the MQTT broker
+        // - If NO payload is returned do nothing. This is for non-standard behaviour
+        //   for e.g. click switches where we need to count number of clicks and detect long presses.
+        const publish = (payload) => {
+            // Add device linkquality.
+            if (data.hasOwnProperty('linkquality')) {
+                payload.linkquality = data.linkquality;
+            }
+
+            if (settings.get().advanced.elapsed) {
+                const now = Date.now();
+                if (this.elapsed[data.device.ieeeAddr]) {
+                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
+                }
+
+                this.elapsed[data.device.ieeeAddr] = now;
+            }
+
+            // Check if we have to debounce
+            if (debounce) {
+                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
+            } else {
+                this.publishEntityState(data.device.ieeeAddr, payload);
+            }
+        };
+
+        const meta = {device: data.device};
+        let payload = {};
+        converters.forEach((converter) => {
+            const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
+            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
+            if (converted) {
+                payload = {...payload, ...converted};
+            }
+        });
+
+        if (Object.keys(payload).length) {
+            publish(payload);
+        }
+    }
+}
+
+module.exports = Receive;

lib/extension/report.js

@@ -0,0 +1,256 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../util/logger');
+const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const debounce = require('debounce');
+const ZigbeeHerdsman = require('zigbee-herdsman');
+
+const defaultConfiguration = {
+    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
+};
+
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+];
+
+const reportKey = 1;
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const clusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // Key is used this.pollDebouncers uniqueness
+        key: 1,
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerID
+        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+    },
+    {
+        key: 2,
+        cluster: {
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+    },
+];
+
+class Report extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.queue = new Set();
+        this.failed = new Set();
+        this.pollDebouncers = {};
+    }
+
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
+        }
+
+        return false;
+    }
+
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
+
+        try {
+            for (const ep of device.endpoints) {
+                for (const [cluster, configuration] of Object.entries(clusters)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
+                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                items.push({...entry});
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
+                        await ep.bind(cluster, this.coordinatorEndpoint);
+                        await ep.configureReporting(cluster, items);
+                        logger.info(
+                            `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                        );
+                    }
+                }
+            }
+
+            // eslint-disable-next-line
+            device.meta.reporting = reportKey;
+        } catch (error) {
+            logger.error(
+                `Failed to setup reporting for '${device.ieeeAddr}' - ${error.stack}`,
+            );
+
+            this.failed.add(device.ieeeAddr);
+        }
+
+        device.save();
+        this.queue.delete(device.ieeeAddr);
+    }
+
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
+
+        const {device, definition} = resolvedEntity;
+        // Handle messages of type endDeviceAnnce and devIncoming.
+        // This message is typically send when a device comes online after being powered off
+        // Ikea TRADFRI tend to forget their reporting after powered off.
+        // Re-setup reporting.
+        // https://github.com/Koenkk/zigbee2mqtt/issues/966
+        if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
+
+        if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
+        if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
+        // Gledopto devices don't support reporting.
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+        return true;
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                this.setupReporting(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            this.setupReporting(resolvedEntity);
+        }
+
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
+                        continue;
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${poll.key}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
+        }
+    }
+}
+
+module.exports = Report;

lib/extension/responder.js

@@ -1,75 +0,0 @@
-const zclId = require('zigbee-herdsman/dist/zcl-id');
-
-const OneJanuary2000 = new Date('January 01, 2000 00:00:00').getTime();
-
-/**
- * This extensions is reponsible for respondig to device requests.
- */
-class ExtensionResponder {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-
-        this.onZclFoundation = this.onZclFoundation.bind(this);
-        this.configured = [];
-    }
-
-    setupOnZclFoundation(device) {
-        if (device && device.epList && !this.configured.includes(device.ieeeAddr)) {
-            this.configured.push(device.ieeeAddr);
-
-            device.epList.forEach((epID) => {
-                const ep = this.zigbee.getEndpoint(device.ieeeAddr, epID);
-                ep.onZclFoundation = this.onZclFoundation;
-            });
-        }
-    }
-
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => this.setupOnZclFoundation(device));
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        this.setupOnZclFoundation(device);
-    }
-
-    readResponse(message, endpoint) {
-        const clusterID = message.clusterid;
-        const cluster = zclId.cluster(clusterID).key;
-        const attributes = message.zclMsg.payload.map((p) => zclId.attr(message.clusterid, p.attrId));
-        const response = [];
-
-        attributes.forEach((attribute) => {
-            if (cluster === 'genTime' && attribute.key === 'time') {
-                const time = Math.round(((new Date()).getTime() - OneJanuary2000) / 1000);
-                response.push(this.createReadResponseRec(clusterID, attribute.value, time));
-            }
-        });
-
-        this.zigbee.publish(
-            endpoint.device.ieeeAddr, 'device', cluster, 'readRsp', 'foundation', response,
-            {direction: 1, seqNum: message.zclMsg.seqNum, disDefaultRsp: 1}, endpoint.epId,
-        );
-    }
-
-    createReadResponseRec(cId, attrId, value) {
-        return {
-            attrId: attrId,
-            status: 0,
-            attrData: value,
-            dataType: zclId.attrType(cId, attrId).value,
-        };
-    }
-
-    onZclFoundation(message, endpoint) {
-        const cmd = message.zclMsg.cmdId;
-
-        if (cmd === 'read') {
-            this.readResponse(message, endpoint);
-        }
-    }
-}
-
-module.exports = ExtensionResponder;

lib/extension/softReset.js

@@ -1,63 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-/**
- * This extensions soft resets the ZNP after a certain timeout.
- */
-class SoftReset {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
-    }
-
-    onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
-        this.resetTimer();
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    resetTimer() {
-        if (this.timeout === 0) {
-            return;
-        }
-
-        this.clearTimer();
-        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
-    }
-
-    handleTimeout() {
-        logger.warn('Soft reset timeout triggered');
-
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.warn('Soft reset failed, trying stop/start');
-                this.zigbee.stop((error) => {
-                    logger.warn('Zigbee stopped');
-                    this.zigbee.start((error) => {
-                        if (error) {
-                            logger.error('Failed to restart!');
-                        }
-                    });
-                });
-            } else {
-                logger.warn('Soft resetted ZNP due to timeout');
-            }
-
-            this.resetTimer();
-        });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        this.resetTimer();
-    }
-}
-
-module.exports = SoftReset;

lib/extension/xiaomi.js

@@ -1,70 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(60);
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-
-/**
- * This extensions handles Xiaomi devices.
- * - Marks Xiaomi devices as online.
- * - Polls Xiaomi routers to keep them awake.
- */
-class Xiaomi {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-    }
-
-    onZigbeeStarted() {
-        // Set all Xiaomi devices to be online, so shepherd won't try
-        // to query info from devices (which would fail because they go to sleep).
-        const devices = this.zigbee.getAllClients();
-        devices.forEach((d) => {
-            if (utils.isXiaomiDevice(d)) {
-                const device = this.zigbee.shepherd.find(d.ieeeAddr, 1);
-                if (device) {
-                    device.getDevice().update({
-                        status: 'online',
-                        joinTime: Math.floor(Date.now() / 1000),
-                    });
-                }
-            }
-        });
-
-        /**
-         * Device availability already pings Xiaomi devices.
-         */
-        if (!settings.get().advanced.availability_timeout) {
-            this.startTimer();
-        } else {
-            logger.debug('Disabled pinging of Xiaomi devices in Xiaomi extension because availability is enabled');
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    ping(ieeeAddr) {
-        this.zigbee.ping(ieeeAddr, 'error', null, 'basic');
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => utils.isXiaomiDevice(d) && utils.isRouter(d) && !utils.isBatteryPowered(d))
-            .forEach((d) => this.ping(d.ieeeAddr));
-    }
-}
-
-module.exports = Xiaomi;

lib/mqtt.js

@@ -2,15 +2,15 @@ const mqtt = require('mqtt');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const fs = require('fs');
+const events = require('events');
 
-class MQTT {
+class MQTT extends events.EventEmitter {
     constructor() {
-        this.onConnect = this.onConnect.bind(this);
+        super();
         this.onMessage = this.onMessage.bind(this);
-        this.messageHandler = null;
     }
 
-    connect(messageHandler, callback) {
+    async connect() {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
@@ -22,30 +22,25 @@ class MQTT {
             },
         };
 
+        if (mqttSettings.version) {
+            options.protocolVersion = mqttSettings.version;
+        }
+
+        if (mqttSettings.keepalive) {
+            logger.debug(`Using MQTT keepalive: ${mqttSettings.keepalive}`);
+            options.keepalive = mqttSettings.keepalive;
+        }
+
         if (mqttSettings.ca) {
             logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
-            const ca = fs.readFileSync(mqttSettings.ca);
-
-            if (ca) {
-                options.ca = ca;
-            } else {
-                logger.error(`Error loading CA certificate for MQTT SSL/TLS configuration.`);
-            }
+            options.ca = fs.readFileSync(mqttSettings.ca);
         }
 
         if (mqttSettings.key && mqttSettings.cert) {
             logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
             logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
-
-            const key = fs.readFileSync(mqttSettings.key);
-            const cert = fs.readFileSync(mqttSettings.cert);
-
-            if (key && cert) {
-                options.key = key;
-                options.cert = cert;
-            } else {
-                logger.error(`Error loading key and/or certificate for MQTT SSL/TLS client authentication.`);
-            }
+            options.key = fs.readFileSync(mqttSettings.key);
+            options.cert = fs.readFileSync(mqttSettings.cert);
         }
 
         if (mqttSettings.user && mqttSettings.password) {
@@ -63,16 +58,6 @@ class MQTT {
             options.rejectUnauthorized = false;
         }
 
-        this.client = mqtt.connect(mqttSettings.server, options);
-
-        // Register callbacks.
-        this.client.on('connect', () => {
-            this.onConnect();
-            callback();
-        });
-
-        this.client.on('message', this.onMessage);
-
         // Set timer at interval to check if connected to MQTT server.
         const interval = 10 * 1000; // seconds * 1000.
         this.connectionTimer = setInterval(() => {
@@ -81,22 +66,25 @@ class MQTT {
             }
         }, interval);
 
-        this.messageHandler = messageHandler;
-    }
+        return new Promise((resolve) => {
+            this.client = mqtt.connect(mqttSettings.server, options);
 
-    disconnect() {
-        clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
+            this.client.on('connect', () => {
+                logger.info('Connected to MQTT server');
+                this.publish('bridge/state', 'online', {retain: true, qos: 0});
+                resolve();
+            });
 
-        this.publish('bridge/state', 'offline', {retain: true, qos: 0}, () => {
-            logger.info('Disconnecting from MQTT server');
-            this.client.end();
+            this.client.on('message', this.onMessage);
         });
     }
 
-    onConnect() {
-        logger.info('Connected to MQTT server');
-        this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    async disconnect() {
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = null;
+        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        logger.info('Disconnecting from MQTT server');
+        this.client.end();
     }
 
     subscribe(topic) {
@@ -104,32 +92,28 @@ class MQTT {
     }
 
     onMessage(topic, message) {
-        if (this.messageHandler) {
-            this.messageHandler(topic, message);
-        }
+        this.emit('message', {topic, message: message.toString()});
     }
 
-    publish(topic, payload, options, callback, base=settings.get().mqtt.base_topic) {
-        topic = `${base}/${topic}`;
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
 
-        if (!this.client || this.client.reconnecting) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
+        topic = `${base}/${topic}`;
+        options = {qos: 0, retain: false, ...options};
+
+        if (!this.isConnected()) {
             logger.error(`Not connected to MQTT server!`);
             logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
             return;
         }
 
         logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
-        this.client.publish(topic, payload, options, callback);
-    }
 
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
+        return new Promise((resolve) => {
+            this.client.publish(topic, payload, options, () => resolve());
+        });
     }
 }
 

lib/state.js

@@ -1,26 +1,22 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
 const fs = require('fs');
-const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
 class State {
-    constructor() {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
-        this.stateChangeListeners = [];
-
-        this.handleSettingsChanged = this.handleSettingsChanged.bind(this);
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -29,31 +25,6 @@ class State {
         // Save the state on every interval
         this.clearTimer();
         this.timer = setInterval(() => this.save(), saveInterval);
-
-        // Listen for on settings changed events.
-        settings.addOnChangeHandler(this.handleSettingsChanged);
-
-        this.checkLastSeen();
-    }
-
-    handleSettingsChanged() {
-        this.checkLastSeen();
-    }
-
-    registerOnStateChangeListener(listener) {
-        this.stateChangeListeners.push(listener);
-    }
-
-    checkLastSeen() {
-        if (settings.get().advanced.last_seen === 'disable') {
-            Object.values(this.state).forEach((s) => {
-                if (s.hasOwnProperty('last_seen')) {
-                    delete s.last_seen;
-                }
-            });
-
-            this.save();
-        }
     }
 
     clearTimer() {
@@ -87,31 +58,33 @@ class State {
         fs.writeFileSync(this.file, json, 'utf8');
     }
 
-    exists(ieeeAddr) {
-        return this.state.hasOwnProperty(ieeeAddr);
+    exists(ID) {
+        return this.state.hasOwnProperty(ID);
     }
 
-    get(ieeeAddr) {
-        return this.state[ieeeAddr];
+    get(ID) {
+        return this.state[ID];
     }
 
-    set(ieeeAddr, state) {
+    set(ID, state, reason=null) {
         const toState = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (toState.hasOwnProperty(property)) {
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
                 delete toState[property];
             }
-        });
+        }
 
-        const fromState = this.state[ieeeAddr];
-        this.stateChangeListeners.forEach((listener) => listener(ieeeAddr, fromState, toState));
+        const fromState = this.state[ID];
 
-        this.state[ieeeAddr] = toState;
+        this.state[ID] = toState;
+
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
-    remove(ieeeAddr) {
-        if (this.exists(ieeeAddr)) {
-            delete this.state[ieeeAddr];
+    remove(ID) {
+        if (this.exists(ID)) {
+            delete this.state[ID];
         }
     }
 }

lib/util/data.js

@@ -1,5 +1,4 @@
 const path = require('path');
-const fs = require('fs');
 
 let dataPath = null;
 
@@ -14,18 +13,8 @@ function load() {
 
 load();
 
-function joinPathStorage(file) {
-    const storagePath = path.join(dataPath, '.storage');
-    if (!fs.existsSync(storagePath)) {
-        fs.mkdirSync(storagePath);
-    }
-
-    return path.join(storagePath, file);
-}
-
 module.exports = {
     joinPath: (file) => path.join(dataPath, file),
-    joinPathStorage: (file) => joinPathStorage(file),
     getPath: () => dataPath,
 
     // For test only.

lib/util/fs.js

@@ -1,16 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-
-function readYaml(file) {
-    return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-}
-
-function readYamlIfExists(file) {
-    return fs.existsSync(file) ? readYaml(file) : null;
-}
-
-function writeYaml(file, content) {
-    fs.writeFileSync(file, yaml.safeDump(content));
-}
-
-module.exports = {readYaml, readYamlIfExists, writeYaml};

lib/util/logger.js

@@ -5,61 +5,89 @@ const path = require('path');
 const fs = require('fs');
 const fx = require('mkdir-recursive');
 const rimraf = require('rimraf');
+const colorizer = winston.format.colorize();
+const assert = require('assert');
+
+// What transports to enable
+const output = settings.get().advanced.log_output;
+
+// Directory to log to
+const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+// Make sure that log directoy exsists when not logging to stdout only
+if (output.includes('file')) {
+    fx.mkdirSync(directory);
+}
 
 // Determine the log level.
 const level = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
 
-// Directoy to log to
-const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
-const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
-
-// Make sure that log directoy exsists
-fx.mkdirSync(directory);
-
-// Custom level
-const levels = {
-    levels: {
-        error: 0,
-        warn: 1,
-        info: 2,
-        debug: 3,
-        trace: 4,
-    },
-    colors: {
-        error: 'red',
-        warn: 'yellow',
-        info: 'green',
-        debug: 'blue',
-        trace: 'magenta',
-    },
+const levelWithCompensatedLength = {
+    'info': 'info ',
+    'error': 'error',
+    'warn': 'warn ',
+    'debug': 'debug',
 };
 
+/* istanbul ignore next */
+const timestampFormat = () => moment().format(settings.get().advanced.timestamp_format);
+
+// Setup default console logger
+const transportsToUse = [
+    new winston.transports.Console({
+        level,
+        silent: !output.includes('console'),
+        format: winston.format.combine(
+            winston.format.timestamp({format: timestampFormat}),
+            winston.format.printf(/* istanbul ignore next */(info) => {
+                let {timestamp, level, message} = info;
+                level = level === 'warning' ? 'warn' : level;
+                const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
+                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+            }),
+        ),
+    }),
+];
+
+// Add file logger when enabled
+// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            let {timestamp, level, message} = info;
+            level = level === 'warning' ? 'warn' : level;
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
+if (output.includes('file')) {
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
 // Create logger
-const logger = new winston.Logger({
-    levels: levels.levels,
-    transports: [
-        new winston.transports.File({
-            filename: path.join(directory, 'log.txt'),
-            json: false,
-            level,
-            maxFiles: 3, // Keep last 3 files
-            maxsize: 10000000, // 10MB
-            timestamp: () => new Date().toLocaleString(),
-        }),
-        new winston.transports.Console({
-            timestamp: () => new Date().toLocaleString(),
-            formatter: (options) =>
-                winston.config.colorize(options.level, '  zigbee2mqtt:' + options.level.toLowerCase()) + ' ' +
-                options.timestamp() + ' ' + (options.message ? options.message : '') +
-                (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' ),
-        }),
-    ],
-});
-
-// Add colors
-winston.addColors(levels.colors);
-
-logger.transports.console.level = level;
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
 
 // Cleanup any old log directory.
 function cleanup() {
@@ -68,7 +96,7 @@ function cleanup() {
 
         let directories = fs.readdirSync(rootDirectory).map((d) => {
             d = path.join(rootDirectory, d);
-            return {path: d, birth: fs.statSync(d).birthtimeMs};
+            return {path: d, birth: fs.statSync(d).mtime};
         });
 
         directories.sort((a, b) => b.birth - a.birth);
@@ -81,6 +109,25 @@ function cleanup() {
 }
 
 logger.cleanup = cleanup;
-logger.directory = directory;
+logger.getLevel = () => transportsToUse[0].level;
+logger.setLevel = (level) => {
+    transportsToUse.forEach((transport) => transport.level = level);
+};
+
+// Print to user what logging is enabled
+if (output.includes('file')) {
+    if (output.includes('console')) {
+        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+    } else {
+        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+    }
+    logger.cleanup();
+} else if (output.includes('console')) {
+    logger.info(`Logging to console only'`);
+}
+
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
 
 module.exports = logger;

lib/util/settings.js

@@ -1,9 +1,11 @@
 const data = require('./data');
-const file = data.joinPath('configuration.yaml');
+const utils = require('./utils');
+const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
 const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
-const fs = require('./fs');
-const onChangeHandlers = [];
+const yaml = require('./yaml');
+const Ajv = require('ajv');
+const ajv = new Ajv({allErrors: true});
 
 const defaults = {
     whitelist: [],
@@ -12,7 +14,9 @@ const defaults = {
     mqtt: {
         include_device_information: false,
     },
-    groups: {},
+    serial: {
+        disable_led: false,
+    },
     device_options: {},
     map_options: {
         graphviz: {
@@ -34,28 +38,31 @@ const defaults = {
             },
         },
     },
-    queue: {
-        delay: 250,
-        simultaneously: 5,
-    },
     experimental: {
-        livolo: false,
-        // json or attribute
+        // json or attribute or attribute_and_json
         output: 'json',
+        new_api: false,
     },
     advanced: {
+        legacy_api: true,
+        log_rotation: true,
+        log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
-        log_level: process.env.DEBUG ? 'debug' : 'info',
+        log_file: 'log.txt',
+        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         baudrate: 115200,
         rtscts: true,
+        adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
         availability_blacklist: [],
+        availability_whitelist: [],
 
         /**
          * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
@@ -101,310 +108,584 @@ const defaults = {
          * Home Assistant status topic
          */
         homeassistant_status_topic: 'hass/status',
+
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
+        /**
+         * Configurable timestampFormat
+         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
+         */
+        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
 };
 
+const schema = {
+    type: 'object',
+    properties: {
+        device_options: {type: 'object'},
+        homeassistant: {type: 'boolean'},
+        permit_join: {type: 'boolean'},
+        mqtt: {
+            type: 'object',
+            properties: {
+                base_topic: {type: 'string'},
+                server: {type: 'string'},
+                keepalive: {type: 'number'},
+                ca: {type: 'string'},
+                key: {type: 'string'},
+                cert: {type: 'string'},
+                user: {type: 'string'},
+                password: {type: 'string'},
+                client_id: {type: 'string'},
+                reject_unauthorized: {type: 'boolean'},
+                include_device_information: {type: 'boolean'},
+                version: {type: 'number'},
+            },
+            required: ['base_topic', 'server'],
+        },
+        serial: {
+            type: 'object',
+            properties: {
+                port: {type: ['string', 'null']},
+                disable_led: {type: 'boolean'},
+                adapter: {type: 'string', enum: ['deconz', 'zstack']},
+            },
+        },
+        ban: {type: 'array', items: {type: 'string'}},
+        whitelist: {type: 'array', items: {type: 'string'}},
+        experimental: {
+            type: 'object',
+            properties: {
+                transmit_power: {type: 'number'},
+            },
+        },
+        advanced: {
+            type: 'object',
+            properties: {
+                legacy_api: {type: 'boolean'},
+                pan_id: {type: 'number'},
+                ext_pan_id: {type: 'array', items: {type: 'number'}},
+                channel: {type: 'number', minimum: 11, maximum: 26},
+                cache_state: {type: 'boolean'},
+                log_rotation: {type: 'boolean'},
+                log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
+                log_output: {type: 'array', items: {type: 'string'}},
+                log_directory: {type: 'string'},
+                log_file: {type: 'string'},
+                baudrate: {type: 'number'},
+                rtscts: {type: 'boolean'},
+                soft_reset_timeout: {type: 'number', minimum: 0},
+                network_key: {type: ['array', 'string'], items: {type: 'number'}},
+                last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
+                elapsed: {type: 'boolean'},
+                availability_timeout: {type: 'number', minimum: 0},
+                availability_blacklist: {type: 'array', items: {type: 'string'}},
+                availability_whitelist: {type: 'array', items: {type: 'string'}},
+                report: {type: 'boolean'},
+                homeassistant_discovery_topic: {type: 'string'},
+                homeassistant_status_topic: {type: 'string'},
+                timestamp_format: {type: 'string'},
+                adapter_concurrent: {type: 'number'},
+            },
+        },
+        map_options: {
+            type: 'object',
+            properties: {
+                graphviz: {
+                    type: 'object',
+                    properties: {
+                        colors: {
+                            type: 'object',
+                            properties: {
+                                fill: {
+                                    type: 'object',
+                                    properties: {
+                                        enddevice: {type: 'string'},
+                                        coordinator: {type: 'string'},
+                                        router: {type: 'string'},
+                                    },
+                                },
+                                font: {
+                                    type: 'object',
+                                    properties: {
+                                        enddevice: {type: 'string'},
+                                        coordinator: {type: 'string'},
+                                        router: {type: 'string'},
+                                    },
+                                },
+                                line: {
+                                    type: 'object',
+                                    properties: {
+                                        active: {type: 'string'},
+                                        inactive: {type: 'string'},
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        devices: {
+            type: 'object',
+            propertyNames: {
+                pattern: '^0x[\\d\\w]{16}$',
+            },
+            patternProperties: {
+                '^.*$': {
+                    type: 'object',
+                    properties: {
+                        friendly_name: {type: 'string'},
+                        retain: {type: 'boolean'},
+                        retention: {type: 'number'},
+                        qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
+                    },
+                    required: ['friendly_name'],
+                },
+            },
+        },
+        groups: {
+            type: 'object',
+            propertyNames: {
+                pattern: '^[\\w].*$',
+            },
+            patternProperties: {
+                '^.*$': {
+                    type: 'object',
+                    properties: {
+                        friendly_name: {type: 'string'},
+                        retain: {type: 'boolean'},
+                        devices: {type: 'array', items: {type: 'string'}},
+                        optimistic: {type: 'boolean'},
+                        qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
+                    },
+                    required: ['friendly_name'],
+                },
+            },
+        },
+    },
+    required: ['homeassistant', 'permit_join', 'mqtt'],
+};
+
+
 let _settings;
-
-function getSettings() {
-    if (!_settings) {
-        _settings = read();
-    }
-
-    if (_settings.hasOwnProperty('advanced') && !_settings.advanced) {
-        delete _settings.advanced;
-    }
-
-    return _settings;
-}
-
-function writeRead() {
-    write();
-    _settings = read();
-    onChangeHandlers.forEach((handler) => handler());
-}
+let _settingsWithDefaults;
 
 function write() {
-    const settings = getSettings();
+    const settings = get();
     const toWrite = objectAssignDeep.noMutate(settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
-    const actual = fs.readYaml(file);
+    const actual = yaml.read(file);
+
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
+    }
+
     if (typeof actual.devices === 'string') {
-        fs.writeYaml(data.joinPath(actual.devices), settings.devices);
+        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
         toWrite.devices = actual.devices;
     }
 
     if (typeof actual.groups === 'string') {
-        fs.writeYaml(data.joinPath(actual.groups), settings.groups);
+        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
         toWrite.groups = actual.groups;
     }
 
-    fs.writeYaml(file, toWrite);
+    yaml.writeIfChanged(file, toWrite);
+
+    _settings = read();
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+}
+
+function validate() {
+    const validate = ajv.compile(schema);
+    const valid = validate(_settings);
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    const postfixes = utils.getEndpointNames();
+
+    // Verify that all friendly names are unique
+    const names = [];
+    const check = (name) => {
+        if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
+        if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
+        if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (name.includes('#') || name.includes('+')) {
+            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+        }
+        names.push(name);
+    };
+
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                throw new Error('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+
+    return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
 
 function read() {
-    const s = fs.readYaml(file);
+    const s = yaml.read(file);
+
+    // Read !secret MQTT username and password if set
+    const interpetValue = (value) => {
+        const re = /!(.*) (.*)/g;
+        const match = re.exec(value);
+        if (match) {
+            const file = data.joinPath(`${match[1]}.yaml`);
+            const key = match[2];
+            return yaml.read(file)[key];
+        } else {
+            return value;
+        }
+    };
+
+    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+        s.mqtt.user = interpetValue(s.mqtt.user);
+        s.mqtt.password = interpetValue(s.mqtt.password);
+    }
+
+    if (s.advanced && s.advanced.network_key) {
+        s.advanced.network_key = interpetValue(s.advanced.network_key);
+    }
 
     // Read devices/groups configuration from separate file.
     if (typeof s.devices === 'string') {
         const file = data.joinPath(s.devices);
-        s.devices = fs.readYamlIfExists(file);
+        s.devices = yaml.readIfExists(file) || {};
     }
 
     if (typeof s.groups === 'string') {
         const file = data.joinPath(s.groups);
-        s.groups = fs.readYamlIfExists(file);
+        s.groups = yaml.readIfExists(file) || {};
     }
 
     return s;
 }
 
+function get() {
+    if (!_settings) {
+        _settings = read();
+    }
+
+    return _settings;
+}
+
+function getWithDefaults() {
+    if (!_settingsWithDefaults) {
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    }
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    return _settingsWithDefaults;
+}
+
 function set(path, value) {
-    let obj = getSettings();
+    let settings = get();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
         if (i === path.length - 1) {
-            obj[key] = value;
+            settings[key] = value;
         } else {
-            if (!obj[key]) {
-                obj[key] = {};
+            if (!settings[key]) {
+                settings[key] = {};
             }
 
-            obj = obj[key];
+            settings = settings[key];
         }
     }
 
-    writeRead();
+    write();
 }
 
-const getDevices = () => getSettings().devices || [];
+function getGroup(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.groups[IDorName];
+    if (byID) {
+        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+    }
 
-const getDevice = (ieeeAddr) => getDevices()[ieeeAddr];
+    for (const [ID, group] of Object.entries(settings.groups)) {
+        if (group.friendly_name === IDorName) {
+            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        }
+    }
 
-const getGroups = () => getSettings().groups || {};
+    return null;
+}
 
-const getGroup = (ID) => {
-    const group = getGroups()[ID];
+function getGroups() {
+    const settings = getWithDefaults();
+    return Object.entries(settings.groups).map(([ID, group]) => {
+        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+    });
+}
 
-    if (group && !group.hasOwnProperty('devices')) {
-        group.devices = [];
+function getGroupThrowIfNotExists(IDorName) {
+    const group = getGroup(IDorName);
+    if (!group) {
+        throw new Error(`Group '${IDorName}' does not exist`);
     }
 
     return group;
-};
+}
 
+function getDevice(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.devices[IDorName];
+    if (byID) {
+        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, device] of Object.entries(settings.devices)) {
+        if (device.friendly_name === IDorName) {
+            return {...device, ID, friendlyName: device.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getDeviceThrowIfNotExists(IDorName) {
+    const device = getDevice(IDorName);
+    if (!device) {
+        throw new Error(`Device '${IDorName}' does not exist`);
+    }
+
+    return device;
+}
+
+function getEntity(IDorName) {
+    const device = getDevice(IDorName);
+    if (device) {
+        return {...device, type: 'device'};
+    }
+
+    const group = getGroup(IDorName);
+    if (group) {
+        return {...group, type: 'group'};
+    }
+
+    return null;
+}
+
+function addDevice(ID) {
+    if (getDevice(ID)) {
+        throw new Error(`Device '${ID}' already exists`);
+    }
+
+    const settings = get();
 
-function addDevice(ieeeAddr) {
-    const settings = getSettings();
     if (!settings.devices) {
         settings.devices = {};
     }
 
-    settings.devices[ieeeAddr] = {friendly_name: ieeeAddr, retain: false};
-    writeRead();
+    settings.devices[ID] = {friendly_name: ID};
+    write();
+    return getDevice(ID);
 }
 
-function whitelistDevice(ieeeAddr) {
-    const settings = getSettings();
+function whitelistDevice(ID) {
+    const settings = get();
     if (!settings.whitelist) {
         settings.whitelist = [];
     }
 
-    settings.whitelist.push(ieeeAddr);
+    if (settings.whitelist.includes(ID)) {
+        throw new Error(`Device '${ID}' already whitelisted`);
+    }
 
-    writeRead();
+    settings.whitelist.push(ID);
+    write();
 }
 
-function banDevice(ieeeAddr) {
-    const settings = getSettings();
+function banDevice(ID) {
+    const settings = get();
     if (!settings.ban) {
         settings.ban = [];
     }
 
-    settings.ban.push(ieeeAddr);
-
-    writeRead();
+    settings.ban.push(ID);
+    write();
 }
 
-function removeDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.devices || !settings.devices[ieeeAddr]) return;
-
-    delete settings.devices[ieeeAddr];
-    writeRead();
+function removeDevice(IDorName) {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = get();
+    delete settings.devices[device.ID];
+    write();
 }
 
-function addGroup(groupName) {
-    const settings = getSettings();
+function addGroup(name, ID=null) {
+    if (getGroup(name) || getDevice(name)) {
+        throw new Error(`friendly_name '${name}' is already in use`);
+    }
+
+    const settings = get();
     if (!settings.groups) {
         settings.groups = {};
     }
 
-    let ID = '1';
-    while (settings.groups.hasOwnProperty(ID)) {
-        ID = (Number.parseInt(ID) + 1).toString();
+    if (ID == null) {
+        // look for free ID
+        ID = '1';
+        while (settings.groups.hasOwnProperty(ID)) {
+            ID = (Number.parseInt(ID) + 1).toString();
+        }
+    } else {
+        // ensure provided ID is not in use
+        ID = ID.toString();
+        if (settings.groups.hasOwnProperty(ID)) {
+            throw new Error(`group id '${ID}' is already in use`);
+        }
     }
 
-    if (Object.values(settings.groups).find((g) => g.friendly_name === groupName)) {
-        return false; // Duplicate group names are not allowed
-    }
+    settings.groups[ID] = {friendly_name: name};
+    write();
 
-    settings.groups[ID] = {friendly_name: groupName};
-    writeRead();
-
-    return true;
+    return getGroup(ID);
 }
 
-function addDeviceToGroup(ID, ieeeAddr) {
-    const settings = getSettings();
-    const group = settings.groups[ID];
+function groupHasDevice(group, keys) {
+    for (const device of group.devices) {
+        const index = keys.indexOf(device);
+        if (index != -1) {
+            return keys[index];
+        }
+    }
+
+    return false;
+}
+
+function addDeviceToGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+
+    const group = settings.groups[groupID];
     if (!group.devices) {
         group.devices = [];
     }
 
-    if (!group.devices.includes(ieeeAddr)) {
-        group.devices.push(ieeeAddr);
-        writeRead();
-        return true;
+    if (!groupHasDevice(group, keys)) {
+        group.devices.push(keys[0]);
+        write();
+    }
+}
+
+function removeDeviceFromGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
+
+    const key = groupHasDevice(group, keys);
+    if (key) {
+        group.devices = group.devices.filter((d) => d != key);
+        write();
+    }
+}
+
+function removeGroup(groupIDorName) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    delete settings.groups[groupID];
+    write();
+}
+
+function changeDeviceOptions(IDorName, newOptions) {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = get();
+    objectAssignDeep(settings.devices[device.ID], newOptions);
+    write();
+}
+
+function changeFriendlyName(IDorName, newName) {
+    if (getGroup(newName) || getDevice(newName)) {
+        throw new Error(`friendly_name '${newName}' is already in use`);
+    }
+
+    const settings = get();
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
     } else {
-        return false;
-    }
-}
-
-function removeDeviceFromGroup(ID, ieeeAddr) {
-    const settings = getSettings();
-    const group = settings.groups[ID];
-
-    if (group.devices && group.devices.includes(ieeeAddr)) {
-        group.devices = group.devices.filter((d) => d != ieeeAddr);
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function removeGroup(name) {
-    const settings = getSettings();
-    if (!settings.groups) return;
-
-    const ID = Object.keys(settings.groups).find((key) => {
-        return settings.groups[key].friendly_name === name;
-    });
-
-    if (ID) {
-        delete settings.groups[ID];
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function getIeeeAddrByFriendlyName(friendlyName) {
-    const entry = Object.entries(getDevices()).find(([ieeeAddr, device]) =>
-        device.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function getGroupIDByFriendlyName(friendlyName) {
-    const entry = Object.entries(getGroups()).find(([ID, group]) =>
-        group.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function changeDeviceOptions(ieeeAddr, newOptions) {
-    const settings = getSettings();
-    const currentOptions = settings.devices[ieeeAddr];
-
-    if (!currentOptions) {
-        return;
+        throw new Error(`Device or group '${IDorName}' does not exist`);
     }
 
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
-        }
-    });
-
-    writeRead();
-}
-
-function changeFriendlyName(old, new_) {
-    const settings = getSettings();
-    const ieeeAddr = getIeeeAddrByFriendlyName(old);
-
-    if (!ieeeAddr) {
-        return false;
-    }
-
-    settings.devices[ieeeAddr].friendly_name = new_;
-    writeRead();
-    return true;
-}
-
-// An entity can be either a group or a device.
-function resolveEntity(ID) {
-    let type = null;
-    let friendlyName = null;
-
-    if (module.exports.getIeeeAddrByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a device.
-        friendlyName = ID;
-        ID = module.exports.getIeeeAddrByFriendlyName(ID);
-        type = 'device';
-    } else if (module.exports.getGroupIDByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a group.
-        friendlyName = ID;
-        ID = Number(module.exports.getGroupIDByFriendlyName(ID));
-        type = 'group';
-    } else if (module.exports.getGroup(ID)) {
-        friendlyName = module.exports.getGroup(ID).friendly_name;
-        ID = Number(ID);
-        type = 'group';
-    } else {
-        // By default it is a device with ID as ID.
-        type = 'device';
-        const device = module.exports.getDevice(ID);
-        friendlyName = device ? device.friendly_name : ID;
-    }
-
-    return {ID, type, friendlyName};
+    write();
 }
 
 module.exports = {
-    get: () => objectAssignDeep.noMutate(defaults, getSettings()),
-    write: () => write(),
-    set: (path, value) => set(path, value),
-
+    validate,
+    get: getWithDefaults,
+    set,
     getDevice,
     getGroup,
     getGroups,
-    getDevices,
-    whitelistDevice: (ieeeAddr) => whitelistDevice(ieeeAddr),
-    banDevice: (ieeeAddr) => banDevice(ieeeAddr),
-    addDevice: (ieeeAddr) => addDevice(ieeeAddr),
-    removeDevice: (ieeeAddr) => removeDevice(ieeeAddr),
-    addGroup: (name) => addGroup(name),
-    removeGroup: (name) => removeGroup(name),
-    addDeviceToGroup: (ID, ieeeAddr) => addDeviceToGroup(ID, ieeeAddr),
-    removeDeviceFromGroup: (ID, ieeeAddr) => removeDeviceFromGroup(ID, ieeeAddr),
+    getEntity,
+    whitelistDevice,
+    banDevice,
+    addDevice,
+    removeDevice,
+    addGroup,
+    removeGroup,
+    addDeviceToGroup,
+    removeDeviceFromGroup,
+    changeDeviceOptions,
+    changeFriendlyName,
 
-    getIeeeAddrByFriendlyName: (friendlyName) => getIeeeAddrByFriendlyName(friendlyName),
-    getGroupIDByFriendlyName: (friendlyName) => getGroupIDByFriendlyName(friendlyName),
-    changeFriendlyName: (old, new_) => changeFriendlyName(old, new_),
-    changeDeviceOptions: (ieeeAddr, options) => changeDeviceOptions(ieeeAddr, options),
-    resolveEntity,
-
-    addOnChangeHandler: (handler) => onChangeHandlers.push(handler),
-
-    // For test
-    _getDefaults: () => {
-        return objectAssignDeep.noMutate(defaults);
+    // For tests only
+    _write: write,
+    _reRead: () => {
+        _settings = read();
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
     },
-    _clear: () => _settings = undefined,
+    _getDefaults: () => defaults,
 };

lib/util/utils.js

@@ -1,5 +1,5 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const equals = require('fast-deep-equal');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -27,9 +27,15 @@ function toLocalISOString(dDate) {
         ':' + pad(tzOffset % 60);
 }
 
-const postfixes = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'l1', 'l2',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom',
+const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
 ];
 
 function flatten(arr) {
@@ -39,71 +45,36 @@ function flatten(arr) {
 }
 
 const forceEndDevice = flatten(
-    ['QBKG03LM', 'QBKG04LM']
-        .map((model) => zigbeeShepherdConverters.devices.find((d) => d.model === model))
-        .map((mappedModel) => mappedModel.zigbeeModel));
+    ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
+        .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
+        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
+        .map((definition) => definition.zigbeeModel));
 
-function getEndpointByEntityID(zigbee, entityID, epName) {
-    const device = zigbee.getDevice(entityID);
-    if (!device) {
-        logger.error(`Failed to find device with entity ID '${entityID}'`);
-        return;
-    }
+async function getZigbee2mqttVersion() {
+    return new Promise((resolve, reject) => {
+        const git = require('git-last-commit');
+        const packageJSON = require('../../package.json');
+        const version = packageJSON.version;
 
-    const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-    if (!mappedDevice) {
-        logger.error(`Device with model ID ${device.modelId} is not supported`);
-        return;
-    }
+        git.getLastCommit((err, commit) => {
+            let commitHash = null;
 
-    let epID = null;
-    if (epName) {
-        if (!mappedDevice.ep) {
-            logger.error(`Device ${mappedDevice.model} doesn't define eps`);
-            return;
-        }
-
-        epID = mappedDevice.ep(device)[epName];
-        if (!epID) {
-            logger.error(`Device ${mappedDevice.model} doesn't have ep named '${epName}'`);
-            return;
-        }
-    } else if (mappedDevice.hasOwnProperty('ep')) {
-        const eps = mappedDevice.ep(device);
-        epID = eps[''] || null;
-    }
-
-    const endpoint = zigbee.getEndpoint(entityID, epID);
-    if (!endpoint) {
-        logger.error(`Failed to retrieve for entity ID ${entityID} and endpoint ID ${epID}`);
-    }
-
-    return endpoint;
-}
-
-function getZigbee2mqttVersion(callback) {
-    const git = require('git-last-commit');
-    const packageJSON = require('../../package.json');
-    const version = packageJSON.version;
-
-    git.getLastCommit((err, commit) => {
-        let commitHash = null;
-
-        if (err) {
-            try {
-                commitHash = require('../.hash.json').hash;
-            } catch (error) {
-                commitHash = 'unknown';
+            if (err) {
+                try {
+                    commitHash = require('../../.hash.json').hash;
+                } catch (error) {
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
             }
-        } else {
-            commitHash = commit.shortHash;
-        }
 
-        callback({commitHash, version});
+            resolve({commitHash, version});
+        });
     });
 }
 
-function formatDate(date, type, _default) {
+function formatDate(date, type) {
     let result;
 
     switch (type) {
@@ -117,36 +88,66 @@ function formatDate(date, type, _default) {
         result = date;
         break;
     default:
-        result = _default;
-        break;
+        throw new Error(`Unsupported type '${type}'`);
     }
 
     return result;
 }
 
+function objectHasProperties(object, properties) {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function equalsPartial(object, expected) {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function getObjectsProperty(objects, key, defaultValue) {
+    for (const object of objects) {
+        if (object.hasOwnProperty(key)) {
+            return object[key];
+        }
+    }
+
+    return defaultValue;
+}
+
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
-    isXiaomiDevice: (device) => {
-        return device.modelId !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufId) &&
-            (!device.manufName || !device.manufName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufId),
-    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelId),
-    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
-    isNumeric: (string) => /^\d+$/.test(string),
-    toLocalISOString: (dDate) => toLocalISOString(dDate),
-    getPostfixes: () => postfixes,
-    formatDate: (date, type, _default=null) => formatDate(date, type, _default),
-    correctDeviceType: (device) => {
-        if (device) {
-            if (forceEndDevice.includes(device.modelId)) {
-                return 'EndDevice';
-            }
-        }
-
-        return device.type;
-    },
-    getEndpointByEntityID,
     getZigbee2mqttVersion,
+    objectHasProperties,
+    getObjectsProperty,
+    getEndpointNames: () => endpointNames,
+    isXiaomiDevice: (device) => {
+        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+    },
+    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
+    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelID),
+    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
+    formatDate: (date, type) => formatDate(date, type),
+    equalsPartial,
+    getResponse,
 };

lib/util/yaml.js

@@ -0,0 +1,44 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const equals = require('fast-deep-equal');
+
+function read(file) {
+    try {
+        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.message =
+                `\n\n\n` +
+                `\t====================================================================\n` +
+                `\tYour YAML file '${file}' is invalid\n` +
+                `\tUse e.g. https://jsonformatter.org/yaml-validator to find and fix the issue.\n` +
+                `\t====================================================================\n` +
+                `\n\n` +
+                error.message;
+        }
+
+
+        throw error;
+    }
+}
+
+function readIfExists(file) {
+    return fs.existsSync(file) ? read(file) : null;
+}
+
+function writeIfChanged(file, content) {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.safeDump(content));
+    }
+}
+
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/zigbeeQueue.js

@@ -1,97 +0,0 @@
-const settings = require('./settings');
-
-const maxSimultaneouslyRunning = settings.get().queue.simultaneously;
-const delay = settings.get().queue.delay;
-const error17Retries = 3;
-const logger = require('./logger');
-
-class ZigbeeQueue {
-    constructor() {
-        this.queue = [];
-        this.active = [];
-        this.timer = null;
-        this.running = false;
-        this.ID = 0;
-    }
-
-    start() {
-        this.running = true;
-        this.resetTimer();
-    }
-
-    push(entityID, func) {
-        this.ID++;
-        this.queue.push({entityID, func, attempts: 0, ID: this.ID});
-        this.log(`Added new job with ID ${this.ID} for ${entityID}`);
-    }
-
-    log(message) {
-        logger.trace(`zigbeeQueue: ${message}`);
-    }
-
-    stopTimer() {
-        if (this.timer) {
-            clearInterval(this.timer);
-            this.timer = null;
-        }
-    }
-
-    resetTimer() {
-        this.stopTimer();
-
-        this.timer = setInterval(() => {
-            this.executeNext();
-        }, delay);
-    }
-
-    stop() {
-        this.stopTimer();
-        this.running = false;
-    }
-
-    handleJobComplete(job, error) {
-        this.log(`Completed job with ID ${job.ID} for ${job.entityID}${error ? ` with error ${error}` : ''}`);
-
-        if (error && error.message === 'rsp error: 17' && job.attempts < error17Retries) {
-            // Error 17 means that the buffer of the ZNP was full,
-            // retry this for a maximum of 3 times.
-            job.attempts++;
-            this.active.splice(this.active.indexOf(job), 1);
-        } else {
-            this.active.splice(this.active.indexOf(job), 1);
-            this.queue.splice(this.queue.indexOf(job), 1);
-        }
-    }
-
-    executeNext() {
-        if (!this.running) {
-            return;
-        }
-
-        const next = this.getNext();
-
-        if (next) {
-            this.active.push(next);
-            this.log(`Executing job with ID ${next.ID} for ${next.entityID}`);
-            next.func((error) => this.handleJobComplete(next, error));
-        }
-    }
-
-    getNext() {
-        if (this.active.length > (maxSimultaneouslyRunning - 1)) {
-            return null;
-        }
-
-        for (let i = 0; i < this.queue.length; i++) {
-            const job = this.queue[i];
-            const activeDeviceJob = this.active.find((j) => j.entityID === job.entityID);
-            if (!activeDeviceJob) {
-                return job;
-            }
-        }
-
-        return null;
-    }
-}
-
-module.exports = ZigbeeQueue;

lib/zapp/cie.js

@@ -1,23 +0,0 @@
-/**
- * This is a fake cie app which makes it possible for some iAS devices to join the network.
- * Based on: https://github.com/zigbeer/zapp-cie
- */
-
-const Ziee = require('zigbee-herdsman/dist/ziee');
-const Zive = require('zigbee-herdsman/dist/zive');
-const cieClusters = new Ziee();
-// Direction-Value 1: input, 2: output, 3: input/output
-
-// IasZone Cluster-Init Direction
-cieClusters.init('ssIasZone', 'dir', {value: 2}); // Client Side(Output)
-
-// Init Command Response Handler
-cieClusters.init('ssIasZone', 'cmdRsps', {
-    enrollReq: (zapp, argObj, cb) => {},
-    statusChangeNotification: (zapp, argObj, cb) => {},
-});
-
-// Create Zive with profId: 0x0104 (Decimal: 260 / HA), devId: 0x0400 (Decimal: 1024 / iasControlIndicatingEquipment)
-// Check the Values by https://github.com/zigbeer/zcl-id/wiki#5-table-of-identifiers
-const cieApp = new Zive({profId: 0x0104, devId: 0x0400, discCmds: []}, cieClusters);
-module.exports = cieApp;

lib/zigbee.js

@@ -1,660 +1,279 @@
-const ZShepherd = require('zigbee-herdsman');
+const ZigbeeHerdsman = require('zigbee-herdsman');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const data = require('./util/data');
+const assert = require('assert');
 const utils = require('./util/utils');
-const ZigbeeQueue = require('./util/zigbeeQueue');
-const cieApp = require('./zapp/cie');
+const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
-const zclId = require('zigbee-herdsman/dist/zcl-id');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
-const advancedSettings = settings.get().advanced;
+const endpointNames = utils.getEndpointNames();
+const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-if (advancedSettings.channel < 11 || advancedSettings.channel > 26) {
-    throw new Error(`'${advancedSettings.channel}' is an invalid channel, use a channel between 11 - 26.`);
-}
-
-const shepherdSettings = {
-    net: {
-        panId: advancedSettings.pan_id,
-        extPanId: advancedSettings.ext_pan_id,
-        channelList: [advancedSettings.channel],
-        precfgkey: settings.get().advanced.network_key,
-    },
-    dbPath: data.joinPath('database.db'),
-    coordBackupPath: data.joinPath('coordinator_backup.json'),
-    sp: {
-        baudRate: advancedSettings.baudrate,
-        rtscts: advancedSettings.rtscts,
-    },
-};
-
-const defaultCfg = {
-    manufSpec: 0,
-    disDefaultRsp: 0,
-};
-
-// Don't print network key.
-const shepherdSettingsLog = objectAssignDeep.noMutate(shepherdSettings);
-shepherdSettingsLog.net.precfgkey = 'HIDDEN';
-
-class Zigbee {
+class Zigbee extends events.EventEmitter {
     constructor() {
-        this.onReady = this.onReady.bind(this);
-        this.onMessage = this.onMessage.bind(this);
-        this.onError = this.onError.bind(this);
-        this.messageHandler = null;
-        this.permitJoinTimer = null;
-
-        this.queue = new ZigbeeQueue();
+        super();
+        this.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler.bind(this);
     }
 
-    start(messageHandler, callback) {
-        logger.info(`Starting zigbee-shepherd`);
-        logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettingsLog)}'`);
+    async start() {
+        logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+            },
+        };
 
-        this.messageHandler = messageHandler;
-        this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
+        const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
+        herdsmanSettingsLog.network.networkKey = 'HIDDEN';
+        logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
-        this.shepherd.start((error) => {
-            if (error) {
-                logger.info(`Error while starting zigbee-shepherd, attempting to fix... (takes 60 seconds) (${error})`);
-                this.shepherd.controller._znp.close((() => null));
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
+        }
 
-                setTimeout(() => {
-                    logger.info(`Starting zigbee-shepherd`);
-                    this.shepherd.start((error) => {
-                        if (error) {
-                            logger.error(`Error while starting zigbee-shepherd! (${error})`);
-                            logger.error(
-                                'Press the reset button on the stick (the one closest to the USB) and start again'
-                            );
-                            callback(error);
-                        } else {
-                            this._handleStarted();
-                            callback(null);
-                        }
-                    });
-                }, utils.secondsToMilliseconds(60));
-            } else {
-                this._handleStarted();
-                callback(null);
-            }
-        });
+        try {
+            herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
+            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
+            await this.herdsman.start();
+        } catch (error) {
+            logger.error(`Error while starting zigbee-herdsman`);
+            throw error;
+        }
 
-        // Register callbacks.
-        this.shepherd.on('ready', this.onReady);
-        this.shepherd.on('ind', this.onMessage);
-        this.shepherd.on('error', this.onError);
-        this._acceptDevIncoming = this._acceptDevIncoming.bind(this);
-        this.shepherd.acceptDevIncoming = this._acceptDevIncoming;
-    }
+        this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
+        this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
+        this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
+        this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
+        this.herdsman.on('message', (data) => this.emit('event', 'message', data));
 
-    _handleStarted() {
-        this.logStartupInfo();
+        logger.info('zigbee-herdsman started');
+        logger.info(`Coordinator firmware version: '${JSON.stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
 
-        this.getAllClients().forEach((device) => {
-            // If set whitelist devices, all other device will be ban or reject to join the network
-            if (settings.get().whitelist.size>0) {
+        for (const device of this.getClients()) {
+            // If a whitelist is used, all other device will be removed from the network.
+            if (settings.get().whitelist.length > 0) {
                 if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklist device is connected (${device.ieeeAddr}), removing...`);
-                    this.removeDevice(device.ieeeAddr, true, () => {});
+                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+                    device.removeFromNetwork();
                 }
-            } else {
-                if (settings.get().ban.includes(device.ieeeAddr)) {
-                    logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
-                    this.removeDevice(device.ieeeAddr, false, () => {});
-                }
-            }
-        });
-
-        this.shepherd.backupCoordinator(() => {});
-    }
-
-    _acceptDevIncoming(devInfo, callback) {
-        logger.debug(
-            `Accept device incoming with ieeeAddr '${devInfo.ieeeAddr}' permit join is '${this.getPermitJoin()}'`
-        );
-        // If set whitelist devices, all other device will be ban or reject to join the network
-        if (settings.get().whitelist.size>0) {
-            if (settings.get().whitelist.includes(devInfo.ieeeAddr)) {
-                logger.info(`whitelist device tried to connect (${devInfo.ieeeAddr})`);
-                callback(null, true);
-            } else {
-                logger.debug(`Not allowing device '${devInfo.ieeeAddr}' to join`);
-                callback(null, false);
-            }
-        } else {
-            if (settings.get().ban.includes(devInfo.ieeeAddr)) {
-                logger.info(`Banned device tried to connect (${devInfo.ieeeAddr})`);
-                callback(null, false);
-            } else {
-                logger.debug(`Allowing device '${devInfo.ieeeAddr}' to join`);
-                callback(null, true);
+            } else if (settings.get().ban.includes(device.ieeeAddr)) {
+                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+                device.removeFromNetwork();
             }
         }
-    }
-
-    logStartupInfo() {
-        logger.info('zigbee-shepherd started');
-        logger.info(`Coordinator firmware version: '${this.getFirmwareVersion()}'`);
-        logger.debug(`zigbee-shepherd info: ${JSON.stringify(this.shepherd.info())}`);
-    }
-
-    getFirmwareVersion() {
-        return this.shepherd.info().firmware.revision;
-    }
-
-    softReset(callback) {
-        this.shepherd.reset('soft', callback);
-    }
-
-    stop(callback) {
-        if (this.permitJoinTimer) {
-            clearInterval(this.permitJoinTimer);
-        }
-
-        this.queue.stop();
-
-        // Backup coordinator
-        this.shepherd.backupCoordinator(() => {
-            this.shepherd.stop((error) => {
-                logger.info('zigbee-shepherd stopped');
-                callback(error);
-            });
-        });
-    }
-
-    onReady() {
-        // Mount cieApp
-        this.shepherd.mount(cieApp, (err, epId) => {
-            if (!err) {
-                logger.debug(`Mounted the cieApp (epId ${epId})`);
-            } else {
-                logger.error(`Failed to mount the cieApp`);
-            }
-        });
 
         // Check if we have to turn off the led
         if (settings.get().serial.disable_led) {
-            this.shepherd.controller.request('UTIL', 'ledControl', {ledid: 3, mode: 0});
+            await this.herdsman.setLED(false);
         }
 
-        // Wait some time before we start the queue, many calls skip this queue which hangs the stick
-        setTimeout(() => {
-            this.queue.start();
-        }, 2000);
-
-        logger.info('zigbee-shepherd ready');
+        // Check if we have to set a transmit power
+        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().experimental.transmit_power;
+            await this.herdsman.setTransmitPower(transmitPower);
+            logger.info(`Set transmit power to '${transmitPower}'`);
+        }
     }
 
-    onError(message) {
-        // This event may appear if zigbee-shepherd cannot decode bad packets (invalid checksum).
-        logger.error(message);
+    async getCoordinatorVersion() {
+        return this.herdsman.getCoordinatorVersion();
     }
 
-    permitJoin(permit, callback) {
-        if (permit) {
-            logger.info('Zigbee: allowing new devices to join.');
-        } else {
+    async reset(type) {
+        await this.herdsman.reset(type);
+    }
+
+    async stop() {
+        await this.herdsman.stop();
+        logger.info('zigbee-herdsman stopped');
+    }
+
+    async permitJoin(permit) {
+        permit ?
+            logger.info('Zigbee: allowing new devices to join.') :
             logger.info('Zigbee: disabling joining new devices.');
-        }
 
-        // In zigbee 3.0 a network automatically closes after 254 seconds.
-        // As a workaround, we enable joining again.
-        if (this.permitJoinTimer) {
-            clearInterval(this.permitJoinTimer);
-        }
-
-        if (permit) {
-            this.permitJoinTimer = setInterval(() => {
-                this.shepherd.permitJoin(255, (error) => {
-                    if (error) {
-                        logger.error('Failed to reenable joining');
-                    } else {
-                        logger.info('Successfully reenabled joining');
-                    }
-                });
-            }, utils.secondsToMilliseconds(160));
-        }
-
-        this.shepherd.permitJoin(permit ? 255 : 0, (error) => {
-            if (error) {
-                logger.info(error);
-            }
-
-            if (callback) {
-                callback();
-            }
-        });
+        await this.herdsman.permitJoin(permit);
     }
 
-    getPermitJoin() {
-        return this.shepherd.controller._permitJoinTime === 255;
+    async getPermitJoin() {
+        return this.herdsman.getPermitJoin();
     }
 
-    getAllClients() {
-        return this.getDevices().filter((device) => device.type !== 'Coordinator');
-    }
-
-    removeDevice(deviceID, ban, callback) {
-        if (ban) {
-            settings.banDevice(deviceID);
-        }
-
-        const friendlyName = this.getDeviceFriendlyName(deviceID);
-
-        this.shepherd.remove(deviceID, {reJoin: true}, (error) => {
-            if (error) {
-                logger.warn(`Failed to remove '${friendlyName}', trying force remove...`);
-                this.forceRemove(deviceID, callback);
-            } else {
-                logger.info(`Removed ${friendlyName}`);
-                callback(null);
-            }
-        });
-    }
-
-    forceRemove(deviceID, callback) {
-        const device = this.shepherd._findDevByAddr(deviceID);
-
-        if (device) {
-            const friendlyName = this.getDeviceFriendlyName(deviceID);
-            return this.shepherd._unregisterDev(device, (error) => {
-                logger.info(`Force removed ${friendlyName}`);
-                callback(error);
-            });
-        } else {
-            logger.warn(`Could not find ${deviceID} for force removal`);
-            callback(true);
-        }
-    }
-
-    onMessage(message) {
-        if (this.messageHandler) {
-            this.messageHandler(message);
-        }
+    getClients() {
+        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
     }
 
     getDevices() {
-        return this.shepherd.list();
+        return this.herdsman.getDevices();
     }
 
-    getDevice(ieeeAddr) {
-        return this.getDevices().find((d) => d.ieeeAddr === ieeeAddr);
+    getDeviceByIeeeAddr(ieeeAddr) {
+        return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
     }
 
-    getDeviceFriendlyName(ieeeAddr) {
-        const device = settings.getDevice(ieeeAddr);
-        return device ? device.friendly_name || ieeeAddr : ieeeAddr;
+    getDevicesByType(type) {
+        return this.herdsman.getDevicesByType(type);
     }
 
-    getCoordinator() {
-        const device = this.getDevices().find((d) => d.type === 'Coordinator');
-        return this.shepherd.find(device.ieeeAddr, 1);
-    }
-
-    getScanable() {
-        return this.getDevices().filter((d) => d.type != 'EndDevice');
-    }
-
-    getGroup(ID) {
-        return this.shepherd.getGroup(ID);
-    }
-
-    getGroupFriendlyName(ID) {
-        let friendlyName = null;
-        friendlyName = settings.getGroup(ID).friendly_name;
-        return (friendlyName ? friendlyName : ID);
-    }
-
-    networkScan(callback) {
-        logger.info('Starting network scan...');
-        const lqiScanList = new Set();
-        const rtgScanList = new Set();
-        const linkMap = [];
-        const routeMap = [];
-
-        // Gather the lqi and the route info into separate lists and only collate them when done.
-        const collateMap = () => {
-            linkMap.sort();
-            logger.debug(`Link map: %j`, linkMap);
-            logger.debug(`Route map: %j`, routeMap);
-            // Merge the routes into the linkMap by matching on 'source|target' link short addresses
-            linkMap.forEach((link) => {
-                routeMap.filter((e) => e.key === link.key).forEach((e) => {
-                    link.routes.push(e.destAddr);
-                });
-                link.routes.sort(function(a, b) {
-                    return a-b;
-                });
-                delete link.key;
-            });
-            const networkMap = {nodes: [], links: linkMap};
-            this.getDevices().forEach((device) => {
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                const deviceType = utils.correctDeviceType(device);
-                const scanfailed = [];
-                if (lqiScanList.has(device.ieeeAddr)) {
-                    scanfailed.push('lqi');
-                }
-                if (rtgScanList.has(device.ieeeAddr)) {
-                    scanfailed.push('rtg');
-                }
-                networkMap.nodes.push({ieeeAddr: device.ieeeAddr, friendlyName: friendlyName, type: deviceType,
-                    nwkAddr: device.nwkAddr, manufName: device.manufName, modelId: device.modelId,
-                    status: device.status, scanfailed: scanfailed});
-            });
-            // Clear remaining devices so they don't process when/if they eventually complete
-            lqiScanList.clear();
-            rtgScanList.clear();
-            logger.debug(`Merged map: %j`, networkMap);
-            callback(networkMap);
-        };
-
-        const processLqiResponse = (error, rsp, targetIeeeAddr, targetNwkAddr) => {
-            if (error) {
-                logger.warn(`Failed network lqi scan for device: '${targetIeeeAddr}' with error: '${error}'`);
-            } else {
-                if (lqiScanList.has(targetIeeeAddr)) {
-                    // Haven't processed this one yet
-                    if (rsp && rsp.status === 0 && rsp.neighborlqilist) {
-                        logger.debug(`lqi scan: '${targetIeeeAddr}' with '${rsp.neighborlqilistcount}' neighbors`);
-                        rsp.neighborlqilist.forEach(function(neighbor) {
-                            // only include active relationships
-                            if (neighbor.relationship <= 3) {
-                                // lqi is measured at receiver so link is from neighbor (source) to scanned router
-                                const key = neighbor.nwkAddr + '|' + targetNwkAddr;
-                                linkMap.push({
-                                    key: key, sourceIeeeAddr: neighbor.extAddr, targetIeeeAddr: targetIeeeAddr,
-                                    sourceNwkAddr: neighbor.nwkAddr, lqi: neighbor.lqi, depth: neighbor.depth,
-                                    relationship: neighbor.relationship, routes: []});
-                            }
-                        });
-                        // Remove from scan list and if both lists are done return the completed network map
-                        lqiScanList.delete(targetIeeeAddr);
-                        if (lqiScanList.size === 0 && rtgScanList === 0) {
-                            logger.info('Network scan completed');
-                            collateMap();
-                        } else {
-                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
-                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
-                        }
-                    } else {
-                        logger.warn(`Empty network lqi scan result for: '${targetIeeeAddr}'`);
-                    }
-                } else {
-                    // This target has already had timeout so don't add to result network map
-                    logger.warn(`Ignoring late network lqi scan result for: '${targetIeeeAddr}'`);
-                }
-            }
-        };
-
-        const processRtgResponse = (error, rsp, sourceIeeeAddr, sourceNwkAddr) => {
-            if (error) {
-                logger.warn(`Failed network rtg scan for device: '${sourceIeeeAddr}' with error: '${error}'`);
-            } else {
-                if (rtgScanList.has(sourceIeeeAddr)) {
-                    // Haven't processed this one yet
-                    if (rsp && rsp.status === 0 && rsp.routingtablelist) {
-                        logger.debug(`rtg scan: '${sourceIeeeAddr}' with '${rsp.routingtablelistcount}' entries`);
-                        rsp.routingtablelist.forEach(function(route) {
-                            if (route.routeStatus === 0) {
-                                const key = sourceNwkAddr + '|' + route.nextHopNwkAddr;
-                                routeMap.push({
-                                    key: key, destAddr: route.destNwkAddr});
-                            }
-                        });
-                        // Remove from scan list and if both lists are done return the completed network map
-                        rtgScanList.delete(sourceIeeeAddr);
-                        if (lqiScanList.size === 0 && rtgScanList.size === 0) {
-                            logger.info('Network scan completed');
-                            collateMap();
-                        } else {
-                            logger.debug(`Outstanding network lqi scans for devices: '${[...lqiScanList].join(' ')}'`);
-                            logger.debug(`Outstanding network rtg scans for devices: '${[...rtgScanList].join(' ')}'`);
-                        }
-                    } else {
-                        logger.warn(`Empty network rtg scan result for: '${sourceIeeeAddr}'`);
-                    }
-                } else {
-                    // This source has already had timeout so don't add to result network map
-                    logger.warn(`Ignoring late network rtg scan result for: '${sourceIeeeAddr}'`);
-                }
-            }
-        };
-
-        // Queue up an lqi scan and an rtg scan for coordinator and each router
-        this.getScanable().forEach((dev) => {
-            logger.debug(`Queing network scans for device: '${dev.ieeeAddr}'`);
-            lqiScanList.add(dev.ieeeAddr);
-            this.queue.push(dev.ieeeAddr, (queueCallback) => {
-                this.shepherd.controller.request('ZDO', 'mgmtLqiReq', {dstaddr: dev.nwkAddr, startindex: 0},
-                    (error, rsp, ieeeAddr, nwkAddr) => {
-                        processLqiResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
-                        queueCallback(error);
-                    });
-            });
-            rtgScanList.add(dev.ieeeAddr);
-            this.queue.push(dev.ieeeAddr, (queueCallback) => {
-                this.shepherd.controller.request('ZDO', 'mgmtRtgReq', {dstaddr: dev.nwkAddr, startindex: 0},
-                    (error, rsp, ieeeAddr, nwkAddr) => {
-                        processRtgResponse(error, rsp, dev.ieeeAddr, dev.nwkAddr);
-                        queueCallback(error);
-                    });
-            });
-        });
-
-        // Wait for all device scans before forcing map with whatever results are already in
-        setTimeout(() => {
-            if (lqiScanList.size === 0 && rtgScanList.size === 0) {
-                logger.info('Network scan timeout no outstanding requests');
-            } else {
-                logger.warn(`Network scan timeout, skipping outstanding lqi scans for '${[...lqiScanList].join(' ')}'`);
-                logger.warn(`Network scan timeout, skipping outstanding rtg scans for '${[...rtgScanList].join(' ')}'`);
-                collateMap();
-            }
-        }, lqiScanList.size * 1000);
-    }
-
-    getEndpoint(ieeeAddr, ep) {
-        // If no ep is given, the first endpoint will be returned
-        // Find device in zigbee-shepherd
-        const device = this.getDevice(ieeeAddr);
-        if (!device || !device.epList || !device.epList.length) {
-            logger.error(`Zigbee cannot determine endpoint for '${ieeeAddr}'`);
-            return null;
-        }
-
-        ep = ep ? ep : device.epList[0];
-        const endpoint = this.shepherd.find(ieeeAddr, ep);
-        return endpoint;
-    }
-
-    publish(entityID, entityType, cid, cmd, cmdType, zclData, cfg=defaultCfg, ep, callback) {
-        let entity = null;
-        let friendlyName = null;
-        if (entityType === 'device') {
-            entity = this.getEndpoint(entityID, ep);
-            friendlyName = this.getDeviceFriendlyName(entityID);
-        } else if (entityType === 'group') {
-            entity = this.getGroup(entityID);
-            friendlyName = this.getGroupFriendlyName(entityID);
-        }
-
-        if (!entity) {
-            logger.error(
-                `Cannot publish message to ${entityType} because '${entityID}' is not known by zigbee-shepherd`
-            );
-            return;
-        }
-
-        this.queue.push(entityID, (queueCallback) => {
-            logger.info(
-                `Zigbee publish to ${entityType} '${friendlyName}', ${cid} - ${cmd} - ` +
-                `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${entity.epId}`
-            );
-
-            const callback_ = (error, rsp) => {
-                if (error) {
-                    logger.error(
-                        `Zigbee publish to ${entityType} '${friendlyName}', ${cid} ` +
-                        `- ${cmd} - ${JSON.stringify(zclData)} ` +
-                        `- ${JSON.stringify(cfg)} - ${ep} ` +
-                        `failed with error ${error}`);
-                }
-
-                if (callback) {
-                    callback(error, rsp);
-                }
-
-                queueCallback(error);
-            };
-
-            if (cmdType === 'functional' && entity.functional) {
-                entity.functional(cid, cmd, zclData, cfg, callback_);
-            } else if (cmdType === 'foundation' && entity.foundation) {
-                entity.foundation(cid, cmd, zclData, cfg, callback_);
-            } else {
-                logger.error(`Unknown zigbee publish cmdType ${cmdType}`);
-            }
-        });
-    }
-
-    ping(ieeeAddr, errorLogLevel='error', cb, mechanism='default') {
-        const friendlyName = this.getDeviceFriendlyName(ieeeAddr);
-        const callback = (error) => {
-            if (error) {
-                logger[errorLogLevel](`Failed to ping '${friendlyName}'`);
-            } else {
-                logger.debug(`Successfully pinged '${friendlyName}'`);
-            }
-
-            if (cb) {
-                cb(error);
-            }
-        };
-
-        if (mechanism === 'default') {
-            const device = this.shepherd._findDevByAddr(ieeeAddr);
-            if (device) {
-                logger.debug(`Ping ${ieeeAddr} (default)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    this.shepherd.controller.checkOnline(device, (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        } else if (mechanism === 'basic') {
-            const endpoint = this.getEndpoint(ieeeAddr, null);
-            if (endpoint) {
-                logger.debug(`Ping ${ieeeAddr} (basic)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    endpoint.foundation('genBasic', 'read', [{attrId: 0}], (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        }
-    }
-
-    bind(ep, cluster, target, callback) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const log = ` '${friendlyName}' - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Binding ${log}`);
-            ep.bind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to bind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully bound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    unbind(ep, cluster, target, callback) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const log = ` '${friendlyName}' - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Unbinding ${log}`);
-            ep.unbind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to unbind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully unbound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    /*
-     * Setup reporting.
-     * Attributes is an array of attribute objects.
-     * each attribute object should contain the following properties:
-     *     attr    the attribute name,
-     *     min     the minimal time between reports in seconds,
-     *     max     the maximum time between reports in seconds,
-     *     change  the minimum amount of change before sending a report
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
      */
-    report(ep, cluster, attributes) {
-        const friendlyName = this.getDeviceFriendlyName(ep.device.ieeeAddr);
-        const cfgArr = attributes.map((attribute) => {
-            const attrId = zclId.attr(cluster, attribute.attr).value;
-            const dataType = zclId.attrType(cluster, attribute.attr).value;
-            return {
-                direction: 0,
-                attrId,
-                dataType,
-                minRepIntval: attribute.min,
-                maxRepIntval: attribute.max,
-                repChange: attribute.change,
-            };
-        });
+    resolveEntity(key) {
+        assert(
+            typeof key === 'string' || typeof key === 'number' ||
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
+        );
 
-        const log=`for '${friendlyName}' - ${cluster} - ${attributes.length}`;
+        if (typeof key === 'string' || typeof key === 'number') {
+            if (typeof key === 'number') {
+                key = key.toString();
+            }
 
-        const configReport = () => {
-            this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-                ep.foundation(cluster, 'configReport', cfgArr, defaultCfg, (error) => {
-                    if (error) {
-                        logger.error(`Failed to setup reporting ${log} - (${error})`);
-                    } else {
-                        logger.debug(`Successfully setup reporting ${log}`);
-                    }
+            if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
+                const coordinator = this.getDevicesByType('Coordinator')[0];
+                return {
+                    type: 'device',
+                    device: coordinator,
+                    endpoint: coordinator.getEndpoint(1),
+                    settings: {friendlyName: 'Coordinator'},
+                    name: 'Coordinator',
+                };
+            }
 
-                    queueCallback(error);
-                });
-            });
-        };
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
+            }
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
+            }
 
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Setup reporting ${log}`);
-
-            ep.bind(cluster, this.getCoordinator(), (error) => {
-                if (error) {
-                    logger.error(`Failed to bind for reporting ${log} - (${error})`);
-                } else {
-                    // Only if binding succeeds, setting-up reporting makes sense.
-                    configReport();
+            const entity = settings.getEntity(key);
+            if (!entity) {
+                return null;
+            } else if (entity.type === 'device') {
+                const device = this.getDeviceByIeeeAddr(entity.ID);
+                if (!device) {
+                    return null;
                 }
 
-                queueCallback(error);
-            });
-        });
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
+                let endpoint;
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
+                    } else {
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
+                        endpoint = device.getEndpoint(endpointID);
+                    }
+                } else if (endpoints && endpoints['default']) {
+                    endpoint = device.getEndpoint(endpoints['default']);
+                } else {
+                    endpoint = device.endpoints[0];
+                }
+
+                return {
+                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
+                };
+            } else {
+                let group = this.getGroupByID(entity.ID);
+                if (!group) group = this.createGroup(entity.ID);
+                return {type: 'group', group, settings: entity, name: entity.friendlyName};
+            }
+        } else if (key.constructor.name === 'Device') {
+            const setting = settings.getEntity(key.ieeeAddr);
+            return {
+                type: 'device',
+                device: key,
+                endpoint: key.endpoints[0],
+                settings: setting,
+                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: setting,
+                name: setting ? setting.friendlyName : key.groupID,
+            };
+        }
+    }
+
+    getGroupByID(ID) {
+        return this.herdsman.getGroupByID(ID);
+    }
+
+    getGroups() {
+        return this.herdsman.getGroups();
+    }
+
+    createGroup(groupID) {
+        return this.herdsman.createGroup(groupID);
+    }
+
+    acceptJoiningDeviceHandler(ieeeAddr) {
+        // If set whitelist devices, all other device will be rejected to join the network
+        if (settings.get().whitelist.length > 0) {
+            if (settings.get().whitelist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+                return true;
+            } else {
+                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                return false;
+            }
+        } else if (settings.get().ban.length > 0) {
+            if (settings.get().ban.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+                return false;
+            } else {
+                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        return this.herdsman.touchlinkFactoryReset();
     }
 }
 

package.json

@@ -1,14 +1,14 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.5.0",
-  "description": "Zigbee to MQTT bridge using zigbee-shepherd",
+  "version": "1.14.0",
+  "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "10"
+    "node": "^10 || ^12 || ^13 || ^14"
   },
   "keywords": [
     "xiaomi",
@@ -20,10 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "coverage": "jest --collect-coverage",
-    "eslint": "node_modules/.bin/eslint .",
+    "test-with-coverage": "jest test --coverage",
+    "eslint": "node_modules/.bin/eslint lib/",
     "start": "node index.js",
-    "test": "jest"
+    "test": "jest test",
+    "test-watch": "jest test --watch"
   },
   "author": "Koen Kanters",
   "license": "GPL-3.0",
@@ -32,26 +33,40 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
+    "ajv": "*",
     "debounce": "*",
+    "fast-deep-equal": "*",
     "git-last-commit": "*",
     "js-yaml": "*",
     "mkdir-recursive": "*",
     "moment": "*",
     "mqtt": "*",
     "object-assign-deep": "*",
-    "queue": "*",
     "rimraf": "*",
     "semver": "*",
-    "winston": "2.4.2",
-    "zigbee-herdsman": "0.1.16",
-    "zigbee-shepherd-converters": "10.1.4",
-    "deep-diff": "*"
+    "winston": "*",
+    "winston-syslog": "*",
+    "zigbee-herdsman": "0.12.94",
+    "zigbee-herdsman-converters": "12.0.110"
   },
   "devDependencies": {
-    "@types/jest": "^24.0.11",
     "eslint": "*",
     "eslint-config-google": "*",
-    "eslint-plugin-jest": "^22.3.0",
-    "jest": "^24.3.1"
+    "eslint-plugin-jest": "*",
+    "jest": "*",
+    "tmp": "*"
+  },
+  "jest": {
+    "coverageThreshold": {
+      "global": {
+        "branches": 100,
+        "functions": 100,
+        "lines": 100,
+        "statements": 100
+      }
+    },
+    "collectCoverageFrom": [
+      "lib/**"
+    ]
   }
 }

scripts/generateNewSupportedDevices.js

@@ -1,10 +1,10 @@
 const fs = require('fs');
 const filename = process.argv[2];
 const text = fs.readFileSync(filename, 'utf8');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
 
 for (const line of text.split('\n')) {
-    const model = zigbeeShepherdConverters.devices.find((d) => d.model === line);
+    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
     console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
 }

scripts/install.sh

@@ -26,7 +26,7 @@ chown -R pi:pi /opt/zigbee2mqtt
 
 echo "Running install. This might take a while and can produce som expected errors"
 cd /opt/zigbee2mqtt || exit
-su pi -c "npm install"
+su pi -c "npm ci"
 
 echo "Creating service file zigbee2mqtt.service"
 service_path="/etc/systemd/system/zigbee2mqtt.service"
@@ -70,4 +70,4 @@ fi
 return 0
 }
 
-[[ "$_" == "$0" ]] && zigbee2mqtt-install-package 
+[[ "$_" == "$0" ]] && zigbee2mqtt-install-package

test/assets/exampleExtension.js

@@ -0,0 +1,11 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/availability.test.js

@@ -0,0 +1,372 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
+
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Availability', () => {
+    let controller;
+
+    function getExtension() {
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        jest.useFakeTimers();
+        settings.set(['advanced', 'availability_timeout'], 10)
+        controller = new Controller();
+        mocksClear.forEach((m) => m.mockClear());
+        await controller.start();
+        await flushPromises();
+    });
+
+    afterEach(async () => {
+        await controller.stop();
+        await flushPromises();
+    })
+
+    it('Should publish availabilty on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish availabilty offline when ping fails', async () => {
+        MQTT.publish.mockClear();
+        logger.error.mockClear();
+        logger.debug.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledTimes(3);
+        expect(logger.debug).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish availabilty online and query state on reconnect', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+    });
+
+    it('Should fail gracefully when quering state after reconnect fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('Device timedout')});
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        logger.debug.mockClear();
+        jest.advanceTimersByTime(11 * 2000);
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to read state of 'bulb_color' after reconnect`);
+    });
+
+    it('Shouldnt ping again when still pinging', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        device.ping.mockImplementationOnce(async () => {await wait(100000000)});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should mark device online when receiving message while offline', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
+        MQTT.publish.mockClear();
+        getExtension().state = {};
+        const payload = {device: zigbeeHerdsman.devices.bulb_color};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout on deviceAnnounce', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+    });
+
+    it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not do anything when message has no device', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
+    });
+
+    it('Should not ping devices blacklisted by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices blacklisted by IEEE address', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should ping forced pingable devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        getExtension().state[device.ieeeAddr] = false;
+        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+    });
+
+    it('Should not read when device has no modelID and reconnects', async () => {
+        const device = zigbeeHerdsman.devices.nomodel;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not read when device has is unsupported', async () => {
+        const device = zigbeeHerdsman.devices.unsupported_router;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should stop pinging device when removed', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        const endpoint = device.getEndpoint(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should publish availability when end device joins', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const payload = {device};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button_double_key/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should mark non-pingable device as non-available when offline for longer than 24 hours', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const defaultLastSeen = device.lastSeen;
+        device.lastSeen = Date.now();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        device.lastSeen = device.lastSeen - (1000 * 60 * 60 * 25);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        // Shouldn't do anything more when device is removed
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+
+        device.lastSeen = defaultLastSeen;
+    });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+});

test/bind.test.js

@@ -0,0 +1,203 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+describe('Bind', () => {
+    let controller;
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        MQTT.publish.mockClear();
+    });
+
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
+    });
+
+    it('Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'Coordinator');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'group_1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', '1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch_double/right');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
+    });
+});

test/bridge.test.js

@@ -0,0 +1,208 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const {coordinator, bulb, unsupported} = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported.ieeeAddr);
+
+describe('Bridge', () => {
+    let controller;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        settings.set(['advanced', 'legacy_api'], false);
+        settings.set(['experimental', 'new_api'], true);
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+        MQTT.publish.mockClear();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+    });
+
+    it('Should publish bridge info on startup', async () => {
+        const version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/info',
+          JSON.stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1,"revision":20190425}},"logLevel":"info","permitJoin":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          JSON.stringify([{"ieeeAddress":"0x000b57fffec6a5b2","type":"Router","networkAddress":40369,"supported":true,"friendlyName":"bulb","definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"},"powerSource":"Mains (single phase)","dateCode":null,"interviewing":false,"interviewCompleted":true},{"ieeeAddress":"0x0017880104e45518","type":"EndDevice","networkAddress":6536,"supported":false,"friendlyName":"0x0017880104e45518","definition":null,"powerSource":"Battery","dateCode":null,"interviewing":false,"interviewCompleted":true}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish devices on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/groups',
+          JSON.stringify([{"ID":1,"friendlyName":"group_1","members":[]},{"ID":15071,"friendlyName":"group_tradfri_remote","members":[]},{"ID":99,"friendlyName":99,"members":[]},{"ID":11,"friendlyName":"group_with_tradfri","members":[]},{"ID":2,"friendlyName":"group_2","members":[]}]),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device joined', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device: zigbeeHerdsman.devices.bulb});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceJoined","data":{"friendlyName":"bulb","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event when device interview started', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'started'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"started","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview failed', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'failed'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"failed","ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device interview successful', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.bulb, status: 'successful'});
+        await zigbeeHerdsman.events.deviceInterview({device: zigbeeHerdsman.devices.unsupported, status: 'successful'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"bulb","status":"successful","ieeeAddress":"0x000b57fffec6a5b2","supported":true,"definition":{"model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supports":"on/off, brightness, color temperature"}}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+          'zigbee2mqtt/bridge/event',
+            JSON.stringify({"type":"deviceInterview","data":{"friendlyName":"0x0017880104e45518","status":"successful","ieeeAddress":"0x0017880104e45518","supported":false,"definition":null}}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+            { retain: true, qos: 0 },
+            expect.any(Function)
+        );
+    });
+
+    it('Should publish event and devices when device leaves', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: zigbeeHerdsman.devices.bulb.ieeeAddr});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/event',
+          JSON.stringify({"type":"deviceLeave","data":{"ieeeAddress":"0x000b57fffec6a5b2"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+          expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/info', expect.any(String), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put transaction in response when request is done with transaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false, "transaction": 22}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{"value":false},"status":"ok", "transaction": 22}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should put error in response when request fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error('Failed to connect to adapter')});
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/permitJoin', JSON.stringify({"value": false}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/permitJoin',
+            JSON.stringify({"data":{},"status":"error","error": "Failed to connect to adapter"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Coverage satisfaction', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/random', JSON.stringify({"value": false}));
+        const device = zigbeeHerdsman.devices.bulb;
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+    });
+});

test/bridgeConfig.test.js

@@ -1,174 +0,0 @@
-const BridgeConfig = require('../lib/extension/bridgeConfig');
-const settings = require('../lib/util/settings');
-const fs = require('../lib/util/fs');
-const objectAssignDeep = require('object-assign-deep');
-const data = require('../lib/util/data');
-const configurationFile = data.joinPath('configuration.yaml');
-const utils = require('./utils');
-
-const mqtt = {
-    subscribe: (topic) => {},
-    log: (type, message) => {},
-};
-
-describe('BridgeConfig', () => {
-    let bridgeConfig;
-    const write = (file, json) => fs.writeYaml(file, json);
-    const read = (file) => fs.readYaml(file);
-    const files = new Map();
-
-    beforeAll(() => {
-        utils.stubLogger(jest);
-        jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            throw new Error(`Fake file not found: ${file}`);
-        });
-        jest.spyOn(fs, 'readYamlIfExists').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            return null;
-        });
-        jest.spyOn(fs, 'writeYaml').mockImplementation((file, content) => {
-            files.set(file, objectAssignDeep.noMutate(content));
-        });
-    });
-
-    beforeEach(() => {
-        settings._clear();
-        files.clear();
-        bridgeConfig = new BridgeConfig(null, mqtt, null, null);
-    });
-
-    afterAll(() => {
-        fs.readYaml.mockRestore();
-        fs.readYamlIfExists.mockRestore();
-        fs.writeYaml.mockRestore();
-    });
-
-    it('Setting elapsed false', async () => {
-        write(configurationFile, {advanced: {elapsed: true}});
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/elapsed', 'false');
-        const expected = {
-            advanced: {
-                elapsed: false,
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Add groups when there are none', async () => {
-        write(configurationFile, {});
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/add_group', 'test123');
-        const expected = {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Add groups when there are', async () => {
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/add_group', 'test1234');
-        const expected = {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Remove group', async () => {
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/remove_group', 'test123');
-        const expected = {
-            groups: {
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Get groups', async () => {
-        jest.spyOn(mqtt, 'log').mockImplementation((type, message) => {
-            expect(type).toBe('groups');
-            expect(Object.keys(message)).toHaveLength(2);
-            expect(message['1'].friendly_name).toBe('test1');
-            expect(message['4711'].friendly_name).toBe('test42');
-        });
-
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test1',
-                },
-                '4711': {
-                    friendly_name: 'test42',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/groups', 'whatever');
-    });
-
-    it('Rename device with é', async () => {
-        jest.spyOn(mqtt, 'log').mockImplementation((type, message) => {
-            expect(type).toBe('device_renamed');
-        });
-
-        write(configurationFile, {
-            devices: {
-                '0x12345678': {
-                    friendly_name: 'test123',
-                    retain: false,
-                },
-            },
-        });
-
-        const payload = {
-            old: 'test123',
-            new: 'tést123',
-        };
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/rename', JSON.stringify(payload));
-        const expected = {
-            devices: {
-                '0x12345678': {
-                    friendly_name: 'tést123',
-                    retain: false,
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-});

test/configure.test.js

@@ -0,0 +1,179 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Configure', () => {
+    let controller;
+
+    expectRemoteConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(2);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+
+        const endpoint2 = device.getEndpoint(2);
+        expect(endpoint2.write).toHaveBeenCalledTimes(1);
+        expect(endpoint2.write).toHaveBeenCalledWith("genBasic", {"49": {"type": 25, "value": 11}}, {"disableDefaultResponse": true, "manufacturerCode": 4107});
+        expect(device.meta.configured).toBe(1);
+    }
+
+    expectRemoteNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(0);
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+    });
+
+    it('Should configure on startup', async () => {
+        expectRemoteConfigured();
+    });
+
+    it('Should re-configure when device rejoins', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        await flushPromises();
+        mockClear(device);
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should not configure twice', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure on zigbee message when not configured yet', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should allow to reconfigure manually', async () => {
+        expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Shouldnt manually reconfigure when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
+    });
+
+    it('Should skip reconfigure when device does not require this', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+        await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+    });
+
+    it('Should not configure when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewing = true;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteNotConfigured();
+        device.interviewing = false;
+    });
+
+    it('Should configure when not interviewCompleted', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewCompleted = false;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+        device.interviewCompleted = true;
+    });
+
+    it('Should not configure when already configuring', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(500));
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should configure max 3 times when fails', async () => {
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+});

test/controller.test.js

@@ -1,183 +1,551 @@
-const Controller = require('../lib/controller');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
-const mqtt = require('../lib/mqtt');
-const utils = require('./utils');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
 
 describe('Controller', () => {
     let controller;
-    let mqttPublish;
 
     beforeEach(() => {
-        utils.stubLogger(jest);
-        jest.spyOn(settings, 'getDevice').mockReturnValue({friendly_name: 'test'});
-        mqttPublish = jest.spyOn(mqtt.prototype, 'publish').mockReturnValue(undefined);
+        zigbeeHerdsman.returnDevices.splice(0);
         controller = new Controller();
-        controller.zigbee = {
-            getDevice: () => {
-                return {
-                    modelId: 'TRADFRI bulb E27 CWS opal 600lm',
-                    manufName: 'IKEA',
-                    hwVersion: '1.1',
-                    swBuildId: '2.0',
-                    dateCode: '2018-01-01',
-                };
-            },
-        };
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
     });
 
-    afterEach(() => {
-        jest.restoreAllMocks();
+    it('Start controller', async () => {
+        await controller.start();
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
+        expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
+        expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
+        expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
     });
 
-    describe('Handling zigbee messages', () => {
-        it('Should handle a zigbee message', () => {
-            const device = {ieeeAddr: '0x12345678', modelId: 'TRADFRI bulb E27 CWS opal 600lm'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'devChange', {onOff: 1}, 1);
-            controller.onZigbeeMessage(message);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(mqttPublish.mock.calls[0][1]).toBe(JSON.stringify({state: 'ON'}));
-        });
+    it('Start controller with specific MQTT settings', async () => {
+        const ca = tmp.fileSync().name;
+        fs.writeFileSync(ca, "ca");
+        const key = tmp.fileSync().name;
+        fs.writeFileSync(key, "key");
+        const cert = tmp.fileSync().name;
+        fs.writeFileSync(cert, "cert");
 
-        it('Should handle a zigbee message when include_device_information is set', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: true,
-                },
-                advanced: {
-                    cache_state: false,
-                    last_seen: 'disable',
-                },
-                experimental: {
-                    output: 'json',
-                },
-            });
+        const configuration = {
+            base_topic: "zigbee2mqtt",
+            server: "mqtt://localhost",
+            keepalive: 30,
+            ca, cert, key,
+            password: 'pass',
+            user: 'user1',
+            client_id: 'my_client_id',
+            reject_unauthorized: false,
+            version: 5,
+        }
+        settings.set(['mqtt'], configuration)
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            keepalive: 30,
+            ca: Buffer.from([99, 97]),
+            key: Buffer.from([107, 101, 121]),
+            cert: Buffer.from([99, 101, 114, 116]),
+            password: 'pass',
+            username: 'user1',
+            clientId: 'my_client_id',
+            rejectUnauthorized: false,
+            protocolVersion: 5,
 
-            const payload = {
-                'state': 'ON',
-                'device': {
-                    'ieeeAddr': '0x12345678',
-                    'friendlyName': 'test',
-                    'manufName': 'IKEA',
-                    'modelId': 'TRADFRI bulb E27 CWS opal 600lm',
-                    'hwVersion': '1.1',
-                    'swBuildId': '2.0',
-                    'dateCode': '2018-01-01',
-                },
-            };
-
-            const device = {ieeeAddr: '0x12345678', modelId: 'TRADFRI bulb E27 CWS opal 600lm'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'devChange', {onOff: 1}, 1);
-            controller.onZigbeeMessage(message);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual(payload);
-        });
-
-        it('Should output to json by default', () => {
-            const payload = {temperature: 1, humidity: 2};
-            controller.publishEntityState('0x12345678', payload);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual(payload);
-        });
-
-        it('Should output to attribute', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: false,
-                },
-                advanced: {
-                    cache_state: false,
-                },
-                experimental: {
-                    output: 'attribute',
-                },
-            });
-
-            const payload = {
-                temperature: 1,
-                humidity: 2,
-                state: 'ON',
-                allowedStates: ['ON', 'OFF'],
-                color: {r: 100, g: 0, b: 102, a: 0},
-                nested: {
-                    state: 'OFF',
-                    color: {r: 1, g: 0, b: 2},
-                },
-            };
-            controller.publishEntityState('0x12345678', payload);
-            expect(mqttPublish.mock.calls.map((x) => [x[0], x[1]])).toEqual([
-                ['test/temperature', '1'],
-                ['test/humidity', '2'],
-                ['test/state', 'ON'],
-                ['test/allowedStates', 'ON,OFF'],
-                ['test/color', '100,0,102'],
-                ['test/nested-state', 'OFF'],
-                ['test/nested-color', '1,0,2'],
-            ]);
-        });
-
-        it('Should cache state', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: false,
-                },
-                advanced: {
-                    cache_state: true,
-                },
-                experimental: {
-                    output: 'json',
-                },
-            });
-
-            controller.publishEntityState('0x12345678', {temperature: 1});
-            controller.publishEntityState('0x12345678', {humidity: 2});
-            controller.publishEntityState('0x12345678', {humidity: 5, temperature: 4});
-            controller.publishEntityState('0x12345678', {temperature: 9});
-            controller.publishEntityState('0x12345678', {temperature: 3});
-            expect(mqttPublish).toHaveBeenCalledTimes(5);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual({temperature: 1});
-            expect(JSON.parse(mqttPublish.mock.calls[1][1])).toStrictEqual({temperature: 1, humidity: 2});
-            expect(JSON.parse(mqttPublish.mock.calls[2][1])).toStrictEqual({temperature: 4, humidity: 5});
-            expect(JSON.parse(mqttPublish.mock.calls[3][1])).toStrictEqual({temperature: 9, humidity: 5});
-            expect(JSON.parse(mqttPublish.mock.calls[4][1])).toStrictEqual({temperature: 3, humidity: 5});
-        });
-
-        it('Shouldnt cache state', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: false,
-                },
-                advanced: {
-                    cache_state: false,
-                },
-                experimental: {
-                    output: 'json',
-                },
-            });
-
-            controller.publishEntityState('0x12345678', {temperature: 1});
-            controller.publishEntityState('0x12345678', {humidity: 2});
-            controller.publishEntityState('0x12345678', {temperature: 3});
-            expect(mqttPublish).toHaveBeenCalledTimes(3);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual({temperature: 1});
-            expect(JSON.parse(mqttPublish.mock.calls[1][1])).toStrictEqual({humidity: 2});
-            expect(JSON.parse(mqttPublish.mock.calls[2][1])).toStrictEqual({temperature: 3});
-        });
-
-        it('Should not send empty messages', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: false,
-                },
-                advanced: {
-                    cache_state: true,
-                },
-                experimental: {
-                    output: 'json',
-                },
-            });
-
-            controller.publishEntityState('0x12345678', {});
-            expect(mqttPublish).toHaveBeenCalledTimes(0);
-        });
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
+
+    it('Should generate network_key when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+    });
+
+    it('Start controller should publish cached states', async () => {
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Start controller should not publish cached states when cache_state is false', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Log when MQTT client is unavailable', async () => {
+        jest.useFakeTimers();
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Dont publish to mqtt when client is unavailable', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(2);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"state\":\"ON\",\"brightness\":50,\"color_temp\":370,\"linkquality\":99,\"color\":{\"r\":100,\"g\":50,\"b\":10},\"dummy\":{\"1\":\"yes\",\"2\":\"no\"}}");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Load empty state when state file does not exist', async () => {
+        data.removeState();
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
+    });
+
+    it('Should remove non whitelisted devices on startup', async () => {
+        settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should remove banned devices on startup', async () => {
+        settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
+    it('Start controller fails', async () => {
+        zigbeeHerdsman.start.mockImplementationOnce(() => {throw new Error('failed')});
+        await controller.start();
+        expect(mockExit).toHaveBeenCalledTimes(1);
+    });
+
+    it('Start controller with permit join true', async () => {
+        settings.set(['permit_join'], false);
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+    });
+
+    it('Refuse to start when configuration.yaml is invalid', async () => {
+        settings.set(['permit_join'], 'invalid');
+        await controller.start();
+        expect(logger.error).toHaveBeenCalledWith('Refusing to start, configuration.yaml is not valid, found the following errors:');
+        expect(logger.error).toHaveBeenCalledWith('\t - permit_join should be boolean');
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Start controller with disable_led', async () => {
+        settings.set(['serial', 'disable_led'], true);
+        await controller.start();
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledWith(false);
+    });
+
+    it('Start controller with transmit power', async () => {
+        settings.set(['experimental', 'transmit_power'], 14);
+        await controller.start();
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledWith(14);
+    });
+
+    it('Start controller and stop', async () => {
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(0);
+    });
+
+    it('Start controller and stop', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Start controller adapter disconnects', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await zigbeeHerdsman.events.adapterDisconnected();
+        await flushPromises();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Handle mqtt message', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('dummytopic', 'dummymessage');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
+    });
+
+    it('On zigbee event message', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1`);
+    });
+
+    it('On zigbee event message with group ID', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
+    });
+
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
+    it('On zigbee event message from unkown device should create it', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
+        expect(settings.getDevice(device.ieeeAddr)).toBeNull();
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(settings.getDevice(device.ieeeAddr)).toStrictEqual({"ID": "0x0017880104e45519", "friendlyName": "0x0017880104e45519", "friendly_name": "0x0017880104e45519"});
+    });
+
+    it('On zigbee deviceJoined', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('acceptJoiningDeviceHandler reject banned device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['ban'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['ban'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['ban'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('Shouldnt crash when two device join events are received', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview started', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'started'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview failed', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'failed'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful not supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.unsupported;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee event device announce', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        settings.set(['devices'], {})
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and NOT settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", 'yes', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+
+    it('Publish entity state attribute_json output filtered', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered (device_options)', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state with device information', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'include_device_information'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
+
+        // Unsupported device should have model "unknown"
+        await controller.publishEntityState('unsupported2', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state without retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], false);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state with retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Should publish entity state with expiring retention', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'version'], 5);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retention'], 37);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+    });
+
+    it('Publish entity state no empty messages', async () => {
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Publish should not cache when set', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"state\":\"ON\"}", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"brightness\":200}", {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish should not do anything for unknown entity', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb-unknown', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should start when state is corrupted', async () => {
+        fs.writeFileSync(path.join(data.mockDir, 'state.json'), 'corrupted');
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
+    });
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
 });

test/data.test.js

@@ -1,13 +1,11 @@
+const logger = require('./stub/logger');
 const data = require('../lib/util/data.js');
 const path = require('path');
-const utils = require('./utils');
+const tmp = require('tmp');
+const fs = require('fs');
 
 describe('Data', () => {
     describe('Get path', () => {
-        beforeEach(() => {
-            utils.stubLogger(jest);
-        });
-
         it('Should return correct path', () => {
             const expected = path.normalize(path.join(__dirname, '..', 'data'));
             const actual = data.getPath();
@@ -15,11 +13,12 @@ describe('Data', () => {
         });
 
         it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
-            const expected = path.join('var', 'zigbee2mqtt');
+            const expected = tmp.dirSync().name;
             process.env.ZIGBEE2MQTT_DATA = expected;
             data._reload();
             const actual = data.getPath();
             expect(actual).toBe(expected);
+            expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
             delete process.env.ZIGBEE2MQTT_DATA;
             data._reload();
         });

test/deviceAvailability.test.js

@@ -1,46 +0,0 @@
-const Availability = require('../lib/extension/deviceAvailability');
-const utils = require('./utils');
-
-describe('Availability', () => {
-    let availability;
-
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        availability = new Availability(null, null, null, () => {});
-    });
-
-    afterEach(() => {
-        jest.restoreAllMocks();
-    });
-
-    describe('Determine pingable devices', () => {
-        it('Router device should be a pingable device', () => {
-            const device = {
-                powerSource: 'Mains (single phase)',
-                type: 'Router',
-            };
-
-            expect(availability.isPingable(device)).toBe(true);
-        });
-
-        it('Battery device should not be a pingable device', () => {
-            const device = {
-                powerSource: 'Battery',
-                type: 'EndDevice',
-            };
-
-            expect(availability.isPingable(device)).toBe(false);
-        });
-
-        it('E11-G13 should be a pingable device', () => {
-            const device = {
-                powerSource: 'Mains (single phase)',
-                type: 'EndDevice',
-                modelId: 'E11-G13',
-                manufId: 4448,
-            };
-
-            expect(availability.isPingable(device)).toBe(true);
-        });
-    });
-});

test/deviceBind.test.js

@@ -1,196 +0,0 @@
-const DeviceBind = require('../lib/extension/deviceBind');
-const utils = require('./utils');
-const settings = require('../lib/util/settings');
-
-const mqtt = {
-    subscribe: () => {},
-    log: () => {},
-};
-
-const devices = {
-    bulb: {
-        getSimpleDesc: () => {
-            return {inClusterList: [6, 8]};
-        },
-    },
-    remote: {
-        getSimpleDesc: () => {
-            return {outClusterList: [5, 6, 8]};
-        },
-    },
-    switch_ep2: {
-        getSimpleDesc: () => {
-            return {outClusterList: [5, 6]};
-        },
-    },
-    switch_ep3: {
-        getSimpleDesc: () => {
-            return {inClusterList: [5, 6]};
-        },
-    },
-    occupancy_sensor: {
-        getSimpleDesc: () => {
-            return {outClusterList: [5, 6]};
-        },
-    },
-};
-
-const zigbee = {
-    bind: jest.fn((ep, cluster, target, callback) => {
-        callback(false, null);
-    }),
-    unbind: jest.fn((ep, cluster, target, callback) => {
-        callback(false, null);
-    }),
-    getEndpoint: (ID, ep) => {
-        if (ID === 'bulb' || ID === '0x002') {
-            return devices.bulb;
-        } else if (ID === 'remote' || ID === '0x001') {
-            return devices.remote;
-        } else if (ep == 2 && ID === 'switch_ep2') {
-            return devices.switch_ep2;
-        } else if (ep == 3 && ID === 'switch_ep3') {
-            return devices.switch_ep3;
-        } else if (ep == 2 && ID == 'occupancy_sensor') {
-            return devices.occupancy_sensor;
-        }
-
-        throw new Error(`No mock for ${ID} and ep ${ep}`);
-    },
-    getDevice: (ID) => {
-        const lookup = {
-            'occupancy_sensor': 'SML002',
-            'switch_ep2': 'lumi.sensor_86sw2.es1',
-            'switch_ep3': 'DNCKAT_S003',
-            'bulb': 'TRADFRI bulb E27 WS opal 980lm',
-            'remote': 'TRADFRI remote control',
-            '0x002': 'TRADFRI bulb E27 WS opal 980lm',
-            '0x001': 'TRADFRI remote control',
-        };
-
-        if (lookup.hasOwnProperty(ID)) {
-            return {modelId: lookup[ID]};
-        }
-
-        throw new Error(`No mock for ${ID}`);
-    },
-};
-
-describe('DeviceBind', () => {
-    let deviceBind;
-
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        deviceBind = new DeviceBind(zigbee, mqtt, null, null);
-    });
-
-    afterEach(() => {
-        zigbee.bind.mockClear();
-        zigbee.unbind.mockClear();
-        jest.restoreAllMocks();
-    });
-
-    describe('Bind devices', () => {
-        it('Bind', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/remote', 'bulb');
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind by friendly name', async () => {
-            jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockImplementation((friendlyName) => {
-                const lookup = {
-                    remote: '0x001',
-                    bulb: '0x002',
-                };
-
-                return lookup[friendlyName];
-            });
-
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/remote', 'bulb');
-
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind non default ep', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/switch_ep2/right', 'bulb');
-            expect(zigbee.bind).toHaveBeenCalledTimes(1);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.switch_ep2,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind non default ep to target with non default ep', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/switch_ep2/right', 'switch_ep3/right');
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.switch_ep2,
-                5,
-                devices.switch_ep3,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.switch_ep2,
-                6,
-                devices.switch_ep3,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind default ep when mapped', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/occupancy_sensor', 'bulb');
-            expect(zigbee.bind).toHaveBeenCalledTimes(1);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.occupancy_sensor,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-    });
-
-    describe('Unbind devices', () => {
-        it('Unbind', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/unbind/remote', 'bulb');
-            expect(zigbee.unbind).toHaveBeenCalledTimes(2);
-            expect(zigbee.unbind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.unbind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-    });
-});

test/deviceReceive.test.js

@@ -1,278 +0,0 @@
-const DeviceReceive = require('../lib/extension/deviceReceive');
-const settings = require('../lib/util/settings');
-const devices = require('zigbee-shepherd-converters').devices;
-const utils = require('./utils');
-
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-
-// Devices
-const WXKG11LM = devices.find((d) => d.model === 'WXKG11LM');
-const WXKG02LM = devices.find((d) => d.model === 'WXKG02LM');
-const WSDCGQ11LM = devices.find((d) => d.model === 'WSDCGQ11LM');
-const RTCGQ11LM = devices.find((d) => d.model === 'RTCGQ11LM');
-const ZNCZ02LM = devices.find((d) => d.model === 'ZNCZ02LM');
-
-const mqtt = {
-    log: () => {},
-};
-
-describe('DeviceReceive', () => {
-    let deviceReceive;
-    let publishEntityState;
-
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        jest.spyOn(settings, 'addDevice').mockReturnValue(undefined);
-        publishEntityState = jest.fn();
-        deviceReceive = new DeviceReceive(null, mqtt, null, publishEntityState);
-    });
-
-    afterEach(() => {
-        jest.restoreAllMocks();
-    });
-
-    describe('Handling zigbee messages', () => {
-        it('Should handle a zigbee message', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'single'});
-        });
-
-        it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({homeassistant: true, advanced: {last_seen: 'disable'}});
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(2);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'single'});
-            expect(publishEntityState.mock.calls[1][1]).toStrictEqual({click: ''});
-        });
-
-        it('Should handle a zigbee message which uses ep (left)', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'left'});
-        });
-
-        it('Should handle a zigbee message which uses ep (right)', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 2);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'right'});
-        });
-
-        it('Should handle a zigbee message with default precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.85});
-        });
-
-        it('Should debounce messages', async () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({debounce: 0.1});
-            const message1 = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: 8}, 1
-            );
-            const message2 = utils.zigbeeMessage(
-                device, 'msRelativeHumidity', 'attReport', {measuredValue: 1}, 1
-            );
-            const message3 = utils.zigbeeMessage(
-                device, 'msPressureMeasurement', 'attReport', {measuredValue: 2}, 1
-            );
-            deviceReceive.onZigbeeMessage(message1, device, WSDCGQ11LM);
-            deviceReceive.onZigbeeMessage(message2, device, WSDCGQ11LM);
-            deviceReceive.onZigbeeMessage(message3, device, WSDCGQ11LM);
-            await wait(200);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
-
-            deviceReceive.onZigbeeMessage(message1, device, WSDCGQ11LM);
-            await wait(200);
-            expect(publishEntityState).toHaveBeenCalledTimes(2);
-            expect(publishEntityState.mock.calls[1][1]).toStrictEqual({temperature: 0.08});
-        });
-
-        it('Should handle a zigbee message with 1 precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({temperature_precision: 1});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.8});
-        });
-
-        it('Should handle a zigbee message with 0 precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({temperature_precision: 0});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -1});
-        });
-
-        it('Should handle a zigbee message with 1 precision when set via device_options', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'get').mockReturnValue({
-                device_options: {
-                    temperature_precision: 1,
-                },
-                advanced: {
-                    last_seen: 'disable',
-                },
-            });
-            jest.spyOn(settings, 'getDevice').mockReturnValue({});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.8});
-        }
-        );
-
-        it('Should handle a zigbee message with 2 precision when overrides device_options', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'get').mockReturnValue({
-                device_options: {
-                    temperature_precision: 1,
-                },
-                advanced: {
-                    last_seen: 'disable',
-                },
-            });
-            jest.spyOn(settings, 'getDevice').mockReturnValue({
-                temperature_precision: 2,
-            });
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.85});
-        }
-        );
-
-        it('Should handle a zigbee message with voltage 3010', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 3010}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 100, voltage: 3010};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
-
-        it('Should handle a zigbee message with voltage 2850', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2850}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 35, voltage: 2850};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
-
-        it('Should handle a zigbee message with voltage 2650', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2650}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 14, voltage: 2650};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
-
-        it('Should handle a zigbee message with voltage 2000', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2000}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 0, voltage: 2000};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
-
-        it('Should publish 1 message when converted twice', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const payload = {
-                '65281': {'1': 3045, '3': 19, '4': 17320, '5': 35, '6': [0, 3], '10': 51107, '11': 381, '100': 0},
-            };
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', payload, 1);
-            deviceReceive.onZigbeeMessage(message, device, RTCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {'battery': 100, 'illuminance': 381, 'voltage': 3045};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
-
-        it('Should publish no message when converted without result', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const payload = {'9999': {'1': 3045}};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', payload, 1);
-            deviceReceive.onZigbeeMessage(message, device, RTCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(0);
-        });
-
-        it('Should publish last_seen epoch', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'epoch',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('number');
-        });
-
-        it('Should publish last_seen ISO_8601', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'ISO_8601',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('string');
-        });
-
-        it('Should publish last_seen ISO_8601_local', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'ISO_8601_local',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('string');
-        });
-
-        it('Should not handle messages forwarded Xiaomi messages', () => {
-            const device = {ieeeAddr: '0x12345678', manufId: 4151, type: 'Router'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1, 599);
-            deviceReceive.onZigbeeMessage(message, device, ZNCZ02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(0);
-        });
-
-        it('Should handle messages from Xiaomi router devices', () => {
-            const device = {ieeeAddr: '0x12345678', manufId: 4151, type: 'Router'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1});
-            deviceReceive.onZigbeeMessage(message, device, ZNCZ02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({state: 'ON'});
-        });
-    });
-});

test/group.test.js

@@ -1,287 +1,462 @@
-const Groups = require('../lib/extension/groups');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+const MQTT = require('./stub/mqtt');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
 const settings = require('../lib/util/settings');
-const utils = require('./utils');
-
-let groupExtension = null;
-let zigbee = null;
 
 describe('Groups', () => {
+    let controller;
+
     beforeEach(() => {
-        utils.stubLogger(jest);
-
-        zigbee = {
-            publish: jest.fn(),
-        };
-
-        groupExtension = new Groups(zigbee, null, null, null);
-    });
-
-    afterEach(() => {
-        jest.restoreAllMocks();
-    });
+        data.writeEmptyState();
+        controller = new Controller();
+        Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+    })
 
     it('Apply group updates add', async () => {
-        zigbee.publish.mockClear();
-        const from = {};
-        const to = {'1': ['1', '2']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(2);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '2', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
+        zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
     });
 
     it('Apply group updates remove', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1', '2', '3']};
-        const to = {'1': ['1']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(2);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '2', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
-    it('Apply group updates add 1', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1']};
-        const to = {'1': ['1', '2', '3']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(2);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '2', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
+    it('Move to non existing group', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
-    it('Apply group updates add and remove group', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1', '2']};
-        const to = {'2': ['1', '2']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(4);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '2', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '2', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
+    it('Add non standard endpoint to group with name', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
     });
 
-    it('Apply group updates change 1', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1', '4', '2']};
-        const to = {'1': ['1', '2', '3']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(2);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '4', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
+    it('Add non standard endpoint to group with number', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
     });
 
-    it('Apply group updates change 2', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1', '2', '3']};
-        const to = {'1': ['3', '1', '2', '4']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '4', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-    });
-
-    it('Apply group updates change 3', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1', '2']};
-        const to = {'1': ['3', '1', '4', '2']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(2);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '4', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-    });
-
-    it('Apply group updates change 4', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1']};
-        const to = {'2': ['3', '1']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(3);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
-    });
-
-    it('Apply group updates change 5', async () => {
-        zigbee.publish.mockClear();
-        const from = {'1': ['1']};
-        const to = {'1': ['3'], '2': ['3', '1']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(4);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '1', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '3', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, null, expect.any(Function)
-        );
-    });
-
-    it('Apply group updates add with postfix', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        const from = {};
-        const to = {'1': ['0x12345689/right']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, 3, expect.any(Function)
-        );
-    });
-
-    it('Apply group updates add and remove with postfix', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        const from = {'1': ['0x12345689/right']};
-        const to = {'1': ['0x12345689'], '2': ['0x12345689/left']};
-        groupExtension.apply(from, to);
-        expect(zigbee.publish).toHaveBeenCalledTimes(3);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '2', groupname: ''}, null, 2, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, 3, expect.any(Function)
-        );
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
+    it('Shouldnt crash on non-existing devices', async () => {
+        logger.error.mockClear();
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+        expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
     it('Add to group via MQTT', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
-        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
-        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/add', 'my_switch');
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, null, expect.any(Function)
-        );
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}', {"retain": false, qos: 0}, expect.any(Function));
+
+        // Test if subscribed to topics with slashes
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
     });
 
     it('Add to group via MQTT with postfix', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
-        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
-        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/add', 'my_switch/right');
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'add', 'functional',
-            {groupid: '1', groupname: ''}, null, 3, expect.any(Function)
-        );
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove from group via MQTT with postfix', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
-        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
-        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/remove', 'my_switch/left');
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'remove', 'functional',
-            {groupid: '1'}, null, 2, expect.any(Function)
-        );
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove all group via MQTT', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
-        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/remove_all', 'my_switch');
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'removeAll', 'functional',
-            {}, null, null, expect.any(Function)
-        );
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Remove all group via MQTT deprecated', async () => {
-        zigbee.publish.mockClear();
-        zigbee.getDevice = () => ({modelId: 'lumi.ctrl_neutral2'});
-        zigbee.getEndpoint = (entityID, ep) => ({epId: ep});
-        jest.spyOn(settings, 'getGroupIDByFriendlyName').mockReturnValue(1);
-        jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockReturnValue('0x12345689');
-        groupExtension.onMQTTMessage('zigbee2mqtt/bridge/group/my_group/remove_all', 'my_switch');
-        expect(zigbee.publish).toHaveBeenCalledTimes(1);
-        expect(zigbee.publish).toHaveBeenCalledWith(
-            '0x12345689', 'device', 'genGroups', 'removeAll', 'functional',
-            {}, null, null, expect.any(Function)
-        );
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
     });
-});
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Remove from group all deprecated', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Log when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
+    });
+
+    it('Log when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Device 'bulb_color_not_existing' does not exist");
+    });
+
+    it('Should publish group state change when a device in it changes state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state, filtered', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt publish group state change when a group is not optimistic', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of another group with shared device when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
+            '3': {friendly_name: 'group_3', retain: false, devices: []}
+        });
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via device', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off if all lights within turn off', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off even when missing current state', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller.state.state = {};
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+});

test/legacy/bridgeLegacy.test.js

@@ -0,0 +1,463 @@
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
+const path = require('path');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+
+describe('Bridge legacy', () => {
+    let controller;
+
+    beforeAll(async () => {
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
+        controller = new Controller();
+        await controller.start();
+    })
+
+    beforeEach(() => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+    });
+
+    it('Should publish bridge configuration on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/config',
+          JSON.stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"log_level":'info',"permit_join":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'false');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'wrong');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+    });
+
+    it('Should allow whitelist', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        const bulb = zigbeeHerdsman.devices.bulb;
+        expect(settings.get().whitelist).toStrictEqual([]);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow changing device options', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', "{friendly_name: 'bulb_color'malformed: {retain: true}}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'false');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+    });
+
+    it('Should allow to reset', async () => {
+        zigbeeHerdsman.reset.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledWith('soft');
+        zigbeeHerdsman.reset.mockImplementationOnce(() => {throw new Error('')});
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.reset.mock.calls[1][0]).toBe('soft');
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'disable');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'notvalid');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+    });
+
+    it('Should allow to set log_level', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'error');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'notvalid');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+    });
+
+    it('Should allow to get devices', async () => {
+        const now = Date.now;
+        Date.now = () => 100;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        Date.now = now;
+    });
+
+    it('Should allow to get groups', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/groups', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+    });
+
+    it('Should allow rename devices', async () => {
+        const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(null);
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', "{old: 'bulb_color2'newmalformed: 'bulb_color3'}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+    });
+
+    it('Should allow rename groups', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        await flushPromises();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt rename when no device has been joined', async () => {
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+    });
+
+    it('Should allow to add groups', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', 'new_group');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+    });
+
+    it('Should allow to add groups with json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying only id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
+    });
+
+    it('Should allow to remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt allow add groups without id or friendly_name in json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{}');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith('Failed to add group, missing friendly_name!');
+    });
+
+    it('Shouldnt do anything on unsupported topic', async () => {
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/not_supported', 'to_be_removed');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to force remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromDatabase.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to ban device', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_banned', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.get().ban).toStrictEqual(['0x000b57fffec6a5b3']);
+    });
+
+    it('Shouldnt crash when removing non-existing device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'not_existing_123');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Cannot remove, device 'not_existing_123' does not exist`);
+    });
+
+    it('Should handle when remove fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should handle when ban fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to touchlink factory reset (OK)', async () => {
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
+    });
+
+    it('Should allow to touchlink factory reset (FAILS)', async () => {
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
+    });
+});

test/logger.test.js

@@ -0,0 +1,111 @@
+const tmp = require('tmp');
+const dir = tmp.dirSync();
+let settings;
+const fs = require('fs');
+const path = require('path');
+const data = require('./stub/data');
+let stdOutWriteOriginal;
+const rimraf = require('rimraf');
+
+describe('Logger', () => {
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        jest.resetModules();
+        settings = require('../lib/util/settings');
+        settings.set(['advanced', 'log_directory'], dir.name + '/%TIMESTAMP%');
+        settings._reRead();
+        stdOutWriteOriginal = console._stdout.write;
+        console._stdout.write = () => {};
+    });
+
+    afterEach(async () => {
+        console._stdout.write = stdOutWriteOriginal;
+    });
+
+    it('Create log directory', () => {
+        const logger = require('../lib/util/logger.js');
+        const dirs = fs.readdirSync(dir.name);
+        expect(dirs.length).toBe(1);
+    });
+
+    it('Should cleanup', () => {
+        const logger = require('../lib/util/logger.js');
+
+        for (const d of fs.readdirSync(dir.name)) {
+            rimraf.sync(path.join(dir.name, d));
+        }
+
+        for (let i = 0; i < 21; i++) {
+            fs.mkdirSync(path.join(dir.name, `log_${i}`));
+        }
+
+        expect(fs.readdirSync(dir.name).length).toBe(21);
+        logger.cleanup();
+        expect(fs.readdirSync(dir.name).length).toBe(10);
+    })
+
+    it('Should not cleanup when there is no timestamp set', () => {
+        const logger = require('../lib/util/logger.js');
+        for (let i = 30; i < 40; i++) {
+            fs.mkdirSync(path.join(dir.name, `log_${i}`));
+        }
+
+        settings.set(['advanced', 'log_directory'], dir.name + '/bla');
+        expect(fs.readdirSync(dir.name).length).toBe(20);
+        logger.cleanup();
+        expect(fs.readdirSync(dir.name).length).toBe(20);
+    })
+
+    it('Set and get log level', () => {
+        const logger = require('../lib/util/logger.js');
+        logger.setLevel('debug');
+        expect(logger.getLevel()).toBe('debug');
+    });
+
+    it('Logger should be console and file by default', () => {
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.length).toBe(2);
+        expect(pipes[0].constructor.name).toBe('Console');
+        expect(pipes[0].silent).toBe(false);
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].dirname.startsWith(dir.name)).toBeTruthy();
+    });
+
+    it('Logger can be file only', () => {
+        settings.set(['advanced', 'log_output'], ['file']);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.length).toBe(2);
+        expect(pipes[0].constructor.name).toBe('Console');
+        expect(pipes[0].silent).toBe(true);
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].dirname.startsWith(dir.name)).toBeTruthy();
+    });
+
+    it('Logger can be console only', () => {
+        settings.set(['advanced', 'log_output'], ['console']);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.constructor.name).toBe('Console');
+        expect(pipes.silent).toBe(false);
+    });
+
+    it('Logger can be nothing', () => {
+        settings.set(['advanced', 'log_output'], []);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.constructor.name).toBe('Console');
+        expect(pipes.silent).toBe(true);
+    });
+
+    it('Should allow to disable log rotation', () => {
+        settings.set(['advanced', 'log_rotation'], false);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].maxFiles).toBeNull();
+        expect(pipes[1].tailable).toBeFalsy();
+        expect(pipes[1].maxsize).toBeNull();
+    });
+});

test/networkMap.test.js

@@ -0,0 +1,171 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+
+const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
+
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb_color.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(WXKG02LM.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(CC2530_ROUTER.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported_router.ieeeAddr);
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+Date.now = jest.fn()
+Date.now.mockReturnValue(10000);
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Networkmap', () => {
+    let controller;
+
+    beforeAll(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    function mock() {
+        /**
+         * Topology
+         *
+         *             | -> bulb_color -> unsupported_router (offline)
+         * coordinator |      ^     ^
+         *             |      |     | (not valid)
+         *             | -> bulb    |
+         *                    |  -> CC2530_ROUTER -> WXKG02LM
+         *
+         */
+        coordinator.lqi = () => {return {neighbors: [
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 2, depth: 1, linkquality: 120},
+            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92}
+        ]}};
+        coordinator.routingTable = () => {return {table: [
+            {destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress},
+        ]}};
+
+        bulb.lqi = () => {return {neighbors: [
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 1, depth: 2, linkquality: 110},
+            {ieeeAddr: CC2530_ROUTER.ieeeAddr, networkAddress: CC2530_ROUTER.networkAddress, relationship: 1, depth: 2, linkquality: 100}
+        ]}};
+        bulb.routingTable = () => {return {table: []}};
+
+        bulb_color.lqi = () => {return {neighbors: []}}
+        bulb_color.routingTable = () => {return {table: []}};
+
+        CC2530_ROUTER.lqi = () => {return {neighbors: [
+            {ieeeAddr: WXKG02LM.ieeeAddr, networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
+        ]}};
+        CC2530_ROUTER.routingTable = () => {return {table: []}};
+
+        unsupported_router.lqi = () => {throw new Error('failed')};
+        unsupported_router.routingTable = () => {throw new Error('failed')};
+    }
+
+    it('Output raw networkmap', async () => {
+        mock();
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
+
+        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
+
+        const expected = {
+            "nodes":[
+                {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
+                {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
+                {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
+            ],
+            "links":[
+                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
+                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
+                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
+                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
+                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
+            ]
+        };
+
+        expect(JSON.parse(call[1])).toStrictEqual(expected);
+
+        /**
+         * Check again without routes
+         */
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap', 'raw');
+        await flushPromises();
+        call = MQTT.publish.mock.calls[0];
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
+
+        // Remove routing information
+        expected.nodes.forEach((n) => {
+            if (n.failed && n.failed.includes('routingTable')) {
+                n.failed.splice(n.failed.indexOf('routingTable'), 1);
+            }
+        });
+
+        expected.links.forEach((l) => l.routes = [])
+        expect(JSON.parse(call[1])).toStrictEqual(expected);
+    });
+
+    it('Output graphviz networkmap', async () => {
+        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
+        const getHours = Date.prototype.getHours;
+        Date.prototype.getTimezoneOffset = () => -60;
+        Date.prototype.getHours = () => 1;
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'graphviz');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/graphviz');
+
+        const expected = `digraph G {
+            node[shape=record];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
+              "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
+            }`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+        Date.prototype.getTimezoneOffset = getTimezoneOffset;
+        Date.prototype.getHours = getHours;
+    });
+});

test/onEvent.test.js

@@ -0,0 +1,75 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const mockOnEvent = jest.fn();
+const mappedLivolo = zigbeeHerdsmanConverters.findByDevice(zigbeeHerdsman.devices.LIVOLO);
+mappedLivolo.onEvent = mockOnEvent;
+zigbeeHerdsmanConverters.onEvent = jest.fn();
+
+describe('On event', () => {
+    let controller;
+    const device = zigbeeHerdsman.devices.LIVOLO;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        zigbeeHerdsmanConverters.onEvent.mockClear();
+        await flushPromises();
+    });
+
+    it('Should call with start event', async () => {
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('start')
+        expect(call[1]).toStrictEqual({})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call with stop event', async () => {
+        mockOnEvent.mockClear();
+        await controller.stop();
+        await flushPromises();
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('stop')
+        expect(call[1]).toStrictEqual({})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call with zigbee event', async () => {
+        mockOnEvent.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('deviceAnnounce')
+        expect(call[1]).toStrictEqual({device})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call index onEvent with zigbee event', async () => {
+        zigbeeHerdsmanConverters.onEvent.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
+        const call = zigbeeHerdsmanConverters.onEvent.mock.calls[0];
+        expect(call[0]).toBe('deviceAnnounce')
+        expect(call[1]).toStrictEqual({device})
+        expect(call[2]).toBe(device);
+    });
+});

test/otaUpdate.test.js

@@ -0,0 +1,215 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+describe('OTA update', () => {
+    let controller;
+
+    mockClear = (mapped) => {
+        mapped.ota.updateToLatest = jest.fn();
+        mapped.ota.isUpdateAvailable = jest.fn();
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+    });
+
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+    });
+
+    it('Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Should refuse to check/update when already in progress', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
+            return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
+        });
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
+        jest.runAllTimers();
+        await flushPromises();
+    });
+
+    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
+
+    it('Should check for update when device requests it', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+
+        // Should not request again when device asks again after a short time
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`)
+    });
+
+    it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
+        const device = zigbeeHerdsman.devices.QBKG04LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
+    });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+});

test/receive.test.js

@@ -0,0 +1,453 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Receive', () => {
+    let controller;
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+    });
+
+    it('Should handle a zigbee message', async () => {
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message which uses ep (left)', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message which uses ep (right)', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(2), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message with default precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('Should debounce messages', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        const data1 = {measuredValue: 8}
+        const payload1 = {data: data1, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        const data2 = {measuredValue: 1}
+        const payload2 = {data: data2, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        const data3 = {measuredValue: 2}
+        const payload3 = {data: data3, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload3);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('Should debounce and retain messages when set via device_options', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'debounce'], 0.1);
+        settings.set(['device_options', 'retain'], true);
+        delete settings.get().devices['0x0017880104e45522']['retain'];
+        const data1 = {measuredValue: 8}
+        const payload1 = {data: data1, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        const data2 = {measuredValue: 1}
+        const payload2 = {data: data2, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        const data3 = {measuredValue: 2}
+        const payload3 = {data: data3, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload3);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
+    });
+
+    it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
+        const tempMsg = {data: {measuredValue: 8}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(tempMsg);
+        const pressureMsg = {data: {measuredValue: 2}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(pressureMsg);
+        const tempMsg2 = {data: {measuredValue: 7}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(tempMsg2);
+        const humidityMsg = {data: {measuredValue: 3}, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(humidityMsg);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2, linkquality: 13});
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
+    });
+
+    it('Shouldnt republish old state', async () => {
+        // https://github.com/Koenkk/zigbee2mqtt/issues/3572
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        jest.runAllTimers();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+    });
+
+    it('Should handle a zigbee message with 1 precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('Should handle a zigbee message with 0 precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 0);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('Should handle a zigbee message with 1 precision when set via device_options', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'temperature_precision'], 1);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('Should handle a zigbee message with 2 precision when overrides device_options', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'temperature_precision'], 1);
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 0);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
+
+    it('WSDCGQ11LM pressure precision from non ZCL properties', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
+
+        MQTT.publish.mockClear();
+        let payload = {data: {"65281":{"1":2985,"4":5032,"5":9,"6":[0,1],"10":0,"100":2345,"101":4608,"102":91552}}, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
+
+        MQTT.publish.mockClear();
+        payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+    });
+
+    it('Should handle a zigbee message with voltage 3010', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 3010}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message with voltage 2850', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2850}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message with voltage 2650', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2650}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should handle a zigbee message with voltage 2000', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2000}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish 1 message when converted twice', async () => {
+        const device = zigbeeHerdsman.devices.RTCGQ11LM;
+        const data = {'65281': {'1': 3045, '3': 19, '4': 17320, '5': 35, '6': [0, 3], '10': 51107, '11': 381, '100': 0}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish 1 message when converted twice', async () => {
+        const device = zigbeeHerdsman.devices.RTCGQ11LM;
+        const data = {'9999': {'1': 3045}};
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish last_seen epoch', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish last_seen ISO_8601', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'ISO_8601');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('string')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish last_seen ISO_8601_local', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'ISO_8601_local');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('string')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should not handle messages forwarded Xiaomi messages', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 599};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should handle messages from Xiaomi router devices', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/power_plug');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'state': 'ON', linkquality: 20});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should not handle messages from coordinator', async () => {
+        const device = zigbeeHerdsman.devices.coordinator;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not handle messages from unsupported devices', async () => {
+        const device = zigbeeHerdsman.devices.unsupported;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+        const device = zigbeeHerdsman.devices.interviewing;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+    });
+
+    it('Should handle a command', async () => {
+        const device = zigbeeHerdsman.devices.E1743;
+        const data = {};
+        const payload = {data, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should add elapsed', async () => {
+        settings.set(['advanced', 'elapsed'], true);
+        const device = zigbeeHerdsman.devices.E1743;
+        const payload = {data: {}, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff'};
+        const oldNow = Date.now;
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(new Date(150));
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        Date.now.mockReturnValue(new Date(200));
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop'});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1]).elapsed).toBe(50);
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        Date.now = oldNow;
+    });
+
+    it('Should log when message is from supported device but has no converters', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {inactiveText: 'hello'};
+        const payload = {data, cluster: 'genBinaryOutput', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(logger.debug).toHaveBeenCalledWith('No converter available for \'ZNCZ02LM\' with cluster \'genBinaryOutput\' and type \'attributeReport\' and data \'{"inactiveText":"hello"}\'');
+    });
+
+    it('Should report correct energy and power values for different versions of SP600', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/915, OLD and NEW use different date code
+        // divisor of OLD is not correct and therefore underreports by factor 10.
+        const data = {instantaneousDemand:496,currentSummDelivered:[0,6648]}
+
+        const SP600_NEW = zigbeeHerdsman.devices.SP600_NEW;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+
+        MQTT.publish.mockClear();
+        const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
+    });
+});

test/report.test.js

@@ -0,0 +1,245 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
+
+describe('Report', () => {
+    let controller;
+
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        settings.set(['advanced', 'report'], true);
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    it('Should configure reporting on startup', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        expectOnOffBrightnessColorReport(endpoint, true);
+    });
+
+    it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        device.save.mockClear();
+        mockClear(device);
+        delete device.meta.reporting;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+        expect(device.save).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not configure reporting when still configuring', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(1000));
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        jest.runAllTimers();
+        jest.useRealTimers();
+    });
+
+    it('Should not mark as configured when reporting setup fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
+    it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for end devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure reporting when deviceAnnounce message from IKEA device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+    });
+
+    it('Should not configure reporting on device leave', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for the CC2530 router', async () => {
+        const device = zigbeeHerdsman.devices.CC2530_ROUTER;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting again when it already failed once', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote and should', async () => {
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
+        const device = zigbeeHerdsman.devices.ZNLDP12LM;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+});

test/settings.test.js

@@ -1,322 +1,660 @@
-const objectAssignDeep = require('object-assign-deep');
-
-const data = require('../lib/util/data');
+require('./stub/logger');
+require('./stub/data');
+const data = require('../lib/util/data.js');
+const utils = require('../lib/util/utils.js');
 const settings = require('../lib/util/settings.js');
-const fs = require('../lib/util/fs');
-const utils = require('./utils');
-
+const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
 const groupsFile = data.joinPath('groups.yaml');
+const secretFile = data.joinPath('secret.yaml');
+const yaml = require('js-yaml');
 
 describe('Settings', () => {
-    const write = (file, json) => fs.writeYaml(file, json);
-
-    const read = (file) => fs.readYaml(file);
-
-    const files = new Map();
-
-    beforeAll(() => {
-        utils.stubLogger(jest);
-
-        jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            throw new Error(`Fake file not found: ${file}`);
-        });
-        jest.spyOn(fs, 'readYamlIfExists').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            return null;
-        });
-        jest.spyOn(fs, 'writeYaml').mockImplementation((file, content) => {
-            files.set(file, objectAssignDeep.noMutate(content));
-        });
-    });
+    const write = (file, json, reread=true) => {
+        fs.writeFileSync(file, yaml.safeDump(json))
+        if (reread) {
+            settings._reRead();
+        }
+    };
+    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const remove = (file) => {
+        if (fs.existsSync(file)) fs.unlinkSync(file);
+    }
 
     beforeEach(() => {
-        settings._clear();
-        files.clear();
+        remove(configurationFile);
+        remove(devicesFile);
+        remove(groupsFile);
     });
 
-    afterAll(() => {
-        fs.readYaml.mockRestore();
-        fs.readYamlIfExists.mockRestore();
-        fs.writeYaml.mockRestore();
+    it('Should return default settings', () => {
+        write(configurationFile, {});
+        const s = settings.get();
+        const expected = settings._getDefaults();
+        expected.devices = {};
+        expected.groups = {};
+        expect(s).toStrictEqual(expected);
     });
 
-    describe('Settings', () => {
-        it('Should return default settings', () => {
-            write(configurationFile, {});
-            const s = settings.get();
-            expect(s).toStrictEqual(settings._getDefaults());
-        });
+    it('Should return settings', () => {
+        write(configurationFile, {permit_join: true});
+        const s = settings.get();
+        const expected = settings._getDefaults();
+        expected.devices = {};
+        expected.groups = {};
+        expected.permit_join = true;
+        expect(s).toStrictEqual(expected);
+    });
 
-        it('Should return settings', () => {
-            write(configurationFile, {permit_join: true});
-            const s = settings.get();
-            const expected = settings._getDefaults();
-            expected.permit_join = true;
-            expect(s).toStrictEqual(expected);
-        });
+    it('Should add devices', () => {
+        write(configurationFile, {});
+        settings.addDevice('0x12345678');
 
-        it('Should add devices', () => {
-            write(configurationFile, {});
-            settings.addDevice('0x12345678');
-
-            const actual = read(configurationFile);
-            const expected = {
-                devices: {
-                    '0x12345678': {
-                        friendly_name: '0x12345678',
-                        retain: false,
-                    },
+        const actual = read(configurationFile);
+        const expected = {
+            devices: {
+                '0x12345678': {
+                    friendly_name: '0x12345678',
                 },
-            };
+            },
+        };
 
-            expect(actual).toStrictEqual(expected);
-        });
+        expect(actual).toStrictEqual(expected);
+    });
 
-        it('Should read devices', () => {
-            const content = {
-                devices: {
-                    '0x12345678': {
-                        friendly_name: '0x12345678',
-                        retain: false,
-                    },
+    it('Should read devices', () => {
+        const content = {
+            devices: {
+                '0x12345678': {
+                    friendly_name: '0x12345678',
+                    retain: false,
                 },
-            };
+            },
+        };
 
-            write(configurationFile, content);
+        write(configurationFile, content);
 
-            const device = settings.getDevice('0x12345678');
-            const expected = {
+        const device = settings.getDevice('0x12345678');
+        const expected = {
+            ID: "0x12345678",
+            friendlyName: "0x12345678",
+            friendly_name: '0x12345678',
+            retain: false,
+        };
+
+        expect(device).toStrictEqual(expected);
+    });
+
+    it('Should not throw error when devices is null', () => {
+        const content = {devices: null};
+        write(configurationFile, content);
+        settings.getDevice('0x12345678');
+    });
+
+    it('Should read MQTT username and password form a separate file', () => {
+        const contentConfiguration = {
+            mqtt: {
+                server: 'my.mqtt.server',
+                user: '!secret username',
+                password: '!secret password',
+            },
+            advanced: {
+                network_key: '!secret network_key'
+            }
+        };
+
+        const contentSecret = {
+            username: 'mysecretusername',
+            password: 'mysecretpassword',
+            network_key: [1,2,3],
+        };
+
+        write(secretFile, contentSecret, false);
+        write(configurationFile, contentConfiguration);
+
+        const expected = {
+            include_device_information: false,
+            password: "mysecretpassword",
+            server: "my.mqtt.server",
+            user: "mysecretusername",
+        };
+
+        expect(settings.get().mqtt).toStrictEqual(expected);
+        expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
+
+        settings._write();
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual(contentSecret);
+
+        settings.set(['mqtt', 'user'], 'test123');
+        settings.set(['advanced', 'network_key'], [1,2,3, 4]);
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+        expect(read(secretFile)).toStrictEqual({...contentSecret, username: 'test123', network_key: [1,2,3,4]});
+    });
+
+    it('Should read devices form a separate file', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        const contentDevices = {
+            '0x12345678': {
                 friendly_name: '0x12345678',
                 retain: false,
-            };
+            },
+        };
 
-            expect(device).toStrictEqual(expected);
-        });
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        const device = settings.getDevice('0x12345678');
+        const expected = {
+            ID: "0x12345678",
+            friendlyName: "0x12345678",
+            friendly_name: '0x12345678',
+            retain: false,
+        };
 
-        it('Should read devices form a separate file', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
+        expect(device).toStrictEqual(expected);
+    });
 
-            const contentDevices = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-            };
+    it('Should add devices to a separate file', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
 
-            write(devicesFile, contentDevices);
-            write(configurationFile, contentConfiguration);
-
-            const device = settings.getDevice('0x12345678');
-            const expected = {
+        const contentDevices = {
+            '0x12345678': {
                 friendly_name: '0x12345678',
                 retain: false,
-            };
+            },
+        };
 
-            expect(device).toStrictEqual(expected);
-        });
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
 
-        it('Should add devices to a separate file', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
+        settings.addDevice('0x1234');
 
-            const contentDevices = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-            };
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
 
-            write(devicesFile, contentDevices);
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            const expected = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expected);
-        });
-
-        it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
-
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            const expected = {
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expected);
-        }
-        );
-
-        it('Should add and remove devices to a separate file if devices.yaml doesnt exist', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
-
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            settings.removeDevice('0x1234');
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            expect(read(devicesFile)).toStrictEqual({});
-        }
-        );
-
-        it('Should read groups', () => {
-            const content = {
-                groups: {
-                    '1': {
-                        friendly_name: '123',
-                    },
-                },
-            };
-
-            write(configurationFile, content);
-
-            const group = settings.getGroup('1');
-            const expected = {
-                friendly_name: '123',
-                devices: [],
-            };
-
-            expect(group).toStrictEqual(expected);
-        });
-
-        it('Should read groups from a separate file', () => {
-            const contentConfiguration = {
-                groups: 'groups.yaml',
-            };
-
-            const contentGroups = {
-                '1': {
-                    friendly_name: '123',
-                },
-            };
-
-            write(groupsFile, contentGroups);
-            write(configurationFile, contentConfiguration);
-
-            const group = settings.getGroup('1');
-            const expected = {
-                friendly_name: '123',
-                devices: [],
-            };
-
-            expect(group).toStrictEqual(expected);
-        });
-
-        it('Combine everything! groups and devices from separate file :)', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-                groups: 'groups.yaml',
-            };
-
-            const contentGroups = {
-                '1': {
-                    friendly_name: '123',
-                    devices: [],
-                },
-            };
-
-            write(groupsFile, contentGroups);
-            write(configurationFile, contentConfiguration);
-
-            const expectedConfiguration = {
-                devices: 'devices.yaml',
-                groups: 'groups.yaml',
-            };
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            const expectedDevice = {
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expectedDevice);
-
-            const group = settings.getGroup('1');
-            const expectedGroup = {
-                friendly_name: '123',
-                devices: [],
-            };
-
-            expect(group).toStrictEqual(expectedGroup);
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            const expectedDevice2 = {
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
                 friendly_name: '0x1234',
-                retain: false,
-            };
+            },
+        };
 
-            expect(settings.getDevice('0x1234')).toStrictEqual(expectedDevice2);
+        expect(read(devicesFile)).toStrictEqual(expected);
+    });
+
+    it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        write(configurationFile, contentConfiguration);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        const expected = {
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+    }
+    );
+
+    it('Should add and remove devices to a separate file if devices.yaml doesnt exist', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        write(configurationFile, contentConfiguration);
+
+        settings.addDevice('0x1234');
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        settings.removeDevice('0x1234');
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        expect(read(devicesFile)).toStrictEqual({});
+    }
+    );
+
+    it('Should read groups', () => {
+        const content = {
+            groups: {
+                '1': {
+                    friendly_name: '123',
+                },
+            },
+        };
+
+        write(configurationFile, content);
+
+        const group = settings.getGroup('1');
+        const expected = {
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+            optimistic: true,
+        };
+
+        expect(group).toStrictEqual(expected);
+    });
+
+    it('Should read groups from a separate file', () => {
+        const contentConfiguration = {
+            groups: 'groups.yaml',
+        };
+
+        const contentGroups = {
+            '1': {
+                friendly_name: '123',
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(groupsFile, contentGroups);
+
+        const group = settings.getGroup('1');
+        const expected = {
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+            optimistic: true,
+        };
+
+        expect(group).toStrictEqual(expected);
+    });
+
+    it('Combine everything! groups and devices from separate file :)', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+            groups: 'groups.yaml',
+        };
+
+        const contentGroups = {
+            '1': {
+                friendly_name: '123',
+                devices: [],
+            },
+        };
+        write(configurationFile, contentConfiguration);
+        write(groupsFile, contentGroups);
+
+        const expectedConfiguration = {
+            devices: 'devices.yaml',
+            groups: 'groups.yaml',
+        };
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        const expectedDevice = {
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expectedDevice);
+
+        const group = settings.getGroup('1');
+        const expectedGroup = {
+            optimistic: true,
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+        };
+
+        expect(group).toStrictEqual(expectedGroup);
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        const expectedDevice2 = {
+            ID: '0x1234',
+            friendlyName: '0x1234',
+            friendly_name: '0x1234',
+        };
+
+        expect(settings.getDevice('0x1234')).toStrictEqual(expectedDevice2);
+    });
+
+    it('Should add groups', () => {
+        write(configurationFile, {});
+
+        const added = settings.addGroup('test123');
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should add groups with specific ID', () => {
+        write(configurationFile, {});
+
+        const added = settings.addGroup('test123', 123);
+        const expected = {
+            '123': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should not add duplicate groups', () => {
+        write(configurationFile, {});
+
+        settings.addGroup('test123');
+        expect(() => {
+            settings.addGroup('test123');
+        }).toThrow(new Error("friendly_name 'test123' is already in use"));
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should not add duplicate groups with specific ID', () => {
+        write(configurationFile, {});
+
+        settings.addGroup('test123', 123);
+        expect(() => {
+            settings.addGroup('test_id_123', 123);
+        }).toThrow(new Error("group id '123' is already in use"));
+        const expected = {
+            '123': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should add devices to groups', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            }
         });
 
-        it('Should add groups', () => {
-            write(configurationFile, {});
+        settings.addGroup('test123');
+        settings.addDeviceToGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+                devices: ['0x123'],
+            },
+        };
 
-            const added = settings.addGroup('test123');
-            const expected = {
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should remove devices from groups', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
                 '1': {
                     friendly_name: 'test123',
-                },
-            };
-
-            expect(added).toStrictEqual(true);
-            expect(settings.get().groups).toStrictEqual(expected);
+                    devices: ['0x123'],
+                }
+            }
         });
 
-        it('Should not add duplicate groups', () => {
-            write(configurationFile, {});
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+                devices: [],
+            },
+        };
 
-            const added = settings.addGroup('test123');
-            const added2 = settings.addGroup('test123');
-            const expected = {
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Shouldnt crash when removing device from group when group has no devices', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
                 '1': {
                     friendly_name: 'test123',
-                },
-            };
-
-            expect(added).toStrictEqual(true);
-            expect(added2).toStrictEqual(false);
-            expect(settings.get().groups).toStrictEqual(expected);
+                }
+            }
         });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should throw when adding device to non-existing group', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+        });
+
+        expect(() => {
+            settings.removeDeviceFromGroup('test123', 'bulb')
+        }).toThrow(new Error("Group 'test123' does not exist"));
+    });
+
+    it('Should throw when adding device which already exists', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+        });
+
+        expect(() => {
+            settings.addDevice('0x123')
+        }).toThrow(new Error("Device '0x123' already exists"));
+    });
+
+    it('Should not allow any string values for network_key', () => {
+        write(configurationFile, {
+            advanced: {network_key: 'NOT_GENERATE'},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+    });
+
+
+    it('Should not allow retention configuration without MQTT v5', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError('MQTT retention requires protocol version 5');
+    });
+
+    it('Should not allow non-existing entities in availability blacklist', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
+            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+    });
+
+    it('Should ban devices', () => {
+        write(configurationFile, {});
+        settings.banDevice('0x123');
+        expect(settings.get().ban).toStrictEqual(['0x123']);
+        settings.banDevice('0x1234');
+        expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
+    });
+
+    it('Should throw error when yaml file is invalid', () => {
+        fs.writeFileSync(configurationFile, `
+             good: 9
+             \t wrong
+        `)
+
+        let error;
+        try {
+            settings._reRead();
+        } catch (e) {
+            error = e;
+        }
+
+        expect(error.message).toContain(`Your YAML file '${configurationFile}' is invalid`);
+    });
+
+    it('Should throw error when yaml file does not exist', () => {
+        let error;
+        try {
+            settings._reRead();
+        } catch (e) {
+            error = e;
+        }
+
+        expect(error.message).toContain(`no such file or directory, open`);
+    });
+
+    it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname', retain: false}},
+            groups: {'1': {friendly_name: 'myname', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Duplicate friendly_name 'myname' found`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
+    });
+
+    it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
+        write(configurationFile, {
+            devices: {
+                '0x0017880104e45519': {friendly_name: 'myname', retain: false},
+                '0x0017880104e45511': {friendly_name: 'myname1', retain: false}
+            },
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.changeFriendlyName('myname1', 'myname');
+        }).toThrowError(`friendly_name 'myname' is already in use`);
+    });
+
+    it('Should throw when removing device which doesnt exist', async () => {
+        write(configurationFile, {
+            devices: {
+                '0x0017880104e45519': {friendly_name: 'myname', retain: false},
+                '0x0017880104e45511': {friendly_name: 'myname1', retain: false}
+            },
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.removeDevice('myname33');
+        }).toThrowError(`Device 'myname33' does not exist`);
+    });
+
+    it('Shouldnt write to configuration.yaml when there are no changes in it', () => {
+        const contentConfiguration = {devices: 'devices.yaml'};
+        const contentDevices = {};
+        write(configurationFile, contentConfiguration);
+        const before = fs.statSync(configurationFile).mtimeMs;
+        write(devicesFile, contentDevices);
+        settings.addDevice('0x1234');
+        const after = fs.statSync(configurationFile).mtimeMs;
+        expect(before).toBe(after);
     });
 });

test/stub/data.js

@@ -0,0 +1,222 @@
+const tmp = require('tmp');
+const yaml = require('../../lib/util/yaml');
+const path = require('path');
+const fs = require('fs');
+
+const mockDir = tmp.dirSync().name;
+const mockDirStorage = tmp.dirSync().name;
+
+function writeDefaultConfiguration() {
+    const config = {
+        homeassistant: false,
+        permit_join: true,
+        mqtt: {
+            base_topic: "zigbee2mqtt",
+            server: "mqtt://localhost",
+        },
+        serial: {
+            "port": "/dev/dummy",
+        },
+        devices: {
+            "0x000b57fffec6a5b2": {
+                retain: true,
+                friendly_name: "bulb"
+            },
+            "0x0017880104e45517": {
+                retain: true,
+                friendly_name: "remote"
+            },
+            "0x0017880104e45520": {
+                retain: false,
+                friendly_name: "button"
+            },
+            "0x0017880104e45521": {
+                retain: false,
+                friendly_name: "button_double_key"
+            },
+            "0x0017880104e45522": {
+                qos: 1,
+                retain: false,
+                friendly_name: "weather_sensor"
+            },
+            "0x0017880104e45523": {
+                retain: false,
+                friendly_name: "occupancy_sensor"
+            },
+            "0x0017880104e45524": {
+                retain: false,
+                friendly_name: "power_plug"
+            },
+            "0x0017880104e45530": {
+                retain: false,
+                friendly_name: "button_double_key_interviewing"
+            },
+            "0x0017880104e45540": {
+                friendly_name: "ikea_onoff"
+            },
+            '0x000b57fffec6a5b7': {
+                retain: false,
+                friendly_name: "bulb_2"
+            },
+            "0x000b57fffec6a5b3": {
+                retain: false,
+                friendly_name: "bulb_color"
+            },
+            '0x000b57fffec6a5b4': {
+                retain: false,
+                friendly_name: "bulb_color_2"
+            },
+            "0x0017880104e45541": {
+                retain: false,
+                friendly_name: "wall_switch"
+            },
+            "0x0017880104e45542": {
+                retain: false,
+                friendly_name: "wall_switch_double"
+            },
+            "0x0017880104e45543": {
+                retain: false,
+                friendly_name: "led_controller_1"
+            },
+            "0x0017880104e45544": {
+                retain: false,
+                friendly_name: "led_controller_2"
+            },
+            '0x0017880104e45545': {
+                retain: false,
+                friendly_name: "dimmer_wall_switch"
+            },
+            '0x0017880104e45547': {
+                retain: false,
+                friendly_name: "curtain"
+            },
+            '0x0017880104e45548': {
+                retain: false,
+                friendly_name: 'fan'
+            },
+            '0x0017880104e45549': {
+                retain: false,
+                friendly_name: 'siren'
+            },
+            '0x0017880104e45529': {
+                retain: false,
+                friendly_name: 'unsupported2'
+            },
+            '0x0017880104e45550': {
+                retain: false,
+                friendly_name: 'thermostat'
+            },
+            '0x0017880104e45551': {
+                retain: false,
+                friendly_name: 'smart_vent'
+            },
+            '0x0017880104e45552': {
+                retain: false,
+                friendly_name: 'j1'
+            },
+            '0x0017880104e45553': {
+                retain: false,
+                friendly_name: 'bulb_enddevice'
+            },
+            '0x0017880104e45559': {
+                retain: false,
+                friendly_name: 'cc2530_router'
+            },
+            '0x0017880104e45560': {
+                retain: false,
+                friendly_name: 'livolo'
+            },
+            '0x90fd9ffffe4b64ae': {
+                retain: false,
+                friendly_name: 'tradfri_remote',
+            },
+            '0x90fd9ffffe4b64af': {
+                friendly_name: 'roller_shutter',
+            },
+            '0x90fd9ffffe4b64ax': {
+                friendly_name: 'ZNLDP12LM',
+            },
+            '0x90fd9ffffe4b64aa': {
+                friendly_name: 'SP600_OLD',
+            },
+            '0x90fd9ffffe4b64ab': {
+                friendly_name: 'SP600_NEW',
+            },
+            '0x90fd9ffffe4b64ac': {
+                friendly_name: 'MKS-CM-W5',
+            },
+            '0x0017880104e45526': {
+                friendly_name: 'GL-S-007ZS',
+            },
+            '0x0017880104e43559': {
+                friendly_name: 'U202DST600ZB'
+            },
+            '0x0017880104e44559': {
+                friendly_name: '3157100_thermostat',
+            }
+        },
+        groups: {
+            '1': {
+                friendly_name: 'group_1',
+                retain: false,
+            },
+            '2': {
+                friendly_name: 'group_2',
+                retain: false,
+            },
+            '15071': {
+                friendly_name: 'group_tradfri_remote',
+                retain: false,
+                devices: ['bulb_color_2', 'bulb_2']
+            },
+            '11': {
+                friendly_name: 'group_with_tradfri',
+                retain: false,
+                devices: ['bulb_2']
+            },
+        }
+    };
+
+    yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
+}
+
+function writeEmptyState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
+}
+
+function writeDefaultState() {
+    const state = {
+        "0x000b57fffec6a5b2": {
+            "state": "ON",
+            "brightness": 50,
+            "color_temp": 370,
+            "linkquality": 99,
+        },
+        "0x0017880104e45517": {
+            "brightness": 255
+        },
+    }
+
+    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
+}
+
+function removeState() {
+    fs.unlinkSync(path.join(mockDir, 'state.json'))
+}
+
+jest.mock('../../lib/util/data', () => ({
+    joinPath: (file) => require('path').join(mockDir, file),
+    getPath: () => mockDir,
+}));
+
+writeDefaultConfiguration();
+writeDefaultState();
+
+module.exports = {
+    mockDir,
+    read: () => yaml.read(path.join(mockDir, 'configuration.yaml')),
+    writeDefaultConfiguration,
+    writeDefaultState,
+    removeState,
+    writeEmptyState,
+};

test/stub/logger.js

@@ -0,0 +1,16 @@
+let level = 'info';
+
+const mock = {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    debug: jest.fn(),
+    trace: jest.fn(),
+    cleanup: jest.fn(),
+    setLevel: (newLevel) => {level = newLevel},
+    getLevel: () => level,
+};
+
+jest.mock('../../lib/util/logger', () => (mock));
+
+module.exports = {...mock};

test/stub/mqtt.js

@@ -0,0 +1,24 @@
+const events = {};
+
+const mock = {
+    publish: jest.fn().mockImplementation((topic, payload, options, cb) => cb()),
+    end: jest.fn(),
+    subscribe: jest.fn(),
+    on: (type, handler) => {
+        if (type === 'connect') {
+            handler();
+        }
+
+        events[type] = handler
+    },
+};
+
+const mockConnect = jest.fn().mockReturnValue(mock);
+
+jest.mock('mqtt', () => {
+  return {connect: mockConnect};
+});
+
+module.exports = {
+    events, ...mock, connect: mockConnect,
+};

test/stub/zigbeeHerdsman.js

@@ -0,0 +1,203 @@
+const events = {};
+const assert = require('assert');
+
+class Group {
+    constructor(groupID, members) {
+        this.groupID = groupID;
+        this.command = jest.fn();
+        this.meta = {};
+        this.members = members;
+        this.removeFromDatabase = jest.fn();
+        this.hasMember = (endpoint) => this.members.includes(endpoint);
+    }
+}
+
+const clusters = {
+    'genBasic': 0,
+    'genOta': 25,
+    'genScenes': 5,
+    'genOnOff': 6,
+    'genLevelCtrl': 8,
+    'lightingColorCtrl': 768,
+    'closuresWindowCovering': 258,
+}
+
+class Endpoint {
+    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}) {
+        this.deviceIeeeAddress = deviceIeeeAddress;
+        this.clusterValues = clusterValues;
+        this.ID = ID;
+        this.inputClusters = inputClusters;
+        this.outputClusters = outputClusters;
+        this.command = jest.fn();
+        this.commandResponse = jest.fn();
+        this.read = jest.fn();
+        this.write = jest.fn();
+        this.bind = jest.fn();
+        this.unbind = jest.fn();
+        this.configureReporting = jest.fn();
+        this.binds = binds;
+        this.supportsInputCluster = (cluster) => {
+            assert(clusters[cluster] !== undefined, `Undefined '${cluster}'`);
+            return this.inputClusters.includes(clusters[cluster]);
+        }
+
+        this.supportsOutputCluster = (cluster) => {
+            assert(clusters[cluster], `Undefined '${cluster}'`);
+            return this.outputClusters.includes(clusters[cluster]);
+        }
+
+        this.addToGroup = (group) => {
+            if (!group.members.includes(this)) group.members.push(this);
+        }
+
+        this.getDevice = () => {
+            return Object.values(devices).find(d => d.ieeeAddr === deviceIeeeAddress);
+        }
+
+        this.removeFromGroup = (group) => {
+            const index = group.members.indexOf(this);
+            if (index != -1) {
+                group.members.splice(index, 1);
+            }
+        }
+
+        this.removeFromAllGroups = () => {
+            Object.values(groups).forEach((g) => this.removeFromGroup(g))
+        }
+
+        this.getClusterAttributeValue = (cluster, value) => {
+            return this.clusterValues[cluster][value];
+        }
+    }
+}
+
+class Device {
+    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null) {
+        this.type = type;
+        this.ieeeAddr = ieeeAddr;
+        this.dateCode = dateCode;
+        this.networkAddress = networkAddress;
+        this.manufacturerID = manufacturerID;
+        this.endpoints = endpoints;
+        this.powerSource = powerSource;
+        this.interviewCompleted = interviewCompleted;
+        this.modelID = modelID;
+        this.interviewing = interviewing;
+        this.meta = {};
+        this.ping = jest.fn();
+        this.removeFromNetwork = jest.fn();
+        this.removeFromDatabase = jest.fn();
+        this.save = jest.fn();
+        this.manufacturerName = manufacturerName;
+        this.lastSeen = 1000;
+    }
+
+    getEndpoint(ID) {
+        return this.endpoints.find((e) => e.ID === ID);
+    }
+}
+
+const returnDevices = [];
+
+const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
+const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
+const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
+
+const devices = {
+    'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb_color': bulb_color,
+    'bulb_2': bulb_2,
+    'bulb_color_2': bulb_color_2,
+    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0]}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
+    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
+    'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
+    'WXKG11LM': new Device('EndDevice', '0x0017880104e45520', 6537,4151, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
+    'WXKG02LM': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
+    'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
+    'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
+    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", "lumi.plug"),
+    'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
+    'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
+    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
+    'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
+    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
+    'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
+    'HAMPTON99432':  new Device('Router', '0x0017880104e45548', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'HDC52EastwindFan'),
+    'HS2WD': new Device('Router', '0x0017880104e45549', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'WarningDevice'),
+    '1TST_EU': new Device('Router', '0x0017880104e45550', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'Thermostat'),
+    'SV01': new Device('Router', '0x0017880104e45551', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'SV01-410-MP-1.0'),
+    'J1': new Device('Router', '0x0017880104e45552', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'J1 (5502)'),
+    'E11_G13': new Device('EndDevice', '0x0017880104e45553', 6540,4151, [new Endpoint(1, [0, 6], [])], true, "Mains (single phase)", 'E11-G13'),
+    'nomodel': new Device('Router', '0x0017880104e45535', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Mains (single phase)", undefined, true),
+    'unsupported_router': new Device('Router', '0x0017880104e45525', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Mains (single phase)", "notSupportedModelID", false, "Boef"),
+    'CC2530_ROUTER': new Device('Router', '0x0017880104e45559', 6540,4151, [new Endpoint(1, [0, 6], [])], true, "Mains (single phase)", 'lumi.router'),
+    'LIVOLO': new Device('Router', '0x0017880104e45560', 6541,4152, [new Endpoint(6, [0, 6], [])], true, "Mains (single phase)", 'TI0001          '),
+    'tradfri_remote': new Device('EndDevice', '0x90fd9ffffe4b64ae', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64ae')], true, "Battery", "TRADFRI remote control"),
+    'roller_shutter': new Device('EndDevice', '0x90fd9ffffe4b64af', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64af')], true, "Battery", "SCM-R_00.00.03.15TC"),
+    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "lumi.light.aqcn02"),
+    'SP600_OLD': new Device('Router', '0x90fd9ffffe4b64aa', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20160120'),
+    'SP600_NEW': new Device('Router', '0x90fd9ffffe4b64ab', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20170220'),
+    'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
+    'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
+    'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
+}
+
+const groups = {
+    'group_1': new Group(1, []),
+    'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
+    'group/with/slashes': new Group(99, []),
+    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]])
+}
+
+const mock = {
+    setTransmitPower: jest.fn(),
+    touchlinkFactoryReset: jest.fn(),
+    start: jest.fn(),
+    permitJoin: jest.fn(),
+    getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
+    getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
+    on: (type, handler) => {
+        events[type] = handler;
+    },
+    stop: jest.fn(),
+    setLED: jest.fn(),
+    getDevices: jest.fn().mockImplementation(() => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr));
+    }),
+    getDevicesByType: jest.fn().mockImplementation((type) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).filter((d) => d.type === type);
+    }),
+    getDeviceByIeeeAddr: jest.fn().mockImplementation((ieeeAddr) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.ieeeAddr === ieeeAddr);
+    }),
+    getGroups: jest.fn().mockImplementation((query) => {
+        return Object.values(groups);
+    }),
+    getGroupByID: jest.fn().mockImplementation((groupID) => {
+        return Object.values(groups).find((d) => d.groupID === groupID);
+    }),
+    getPermitJoin: jest.fn().mockReturnValue(false),
+    reset: jest.fn(),
+    createGroup: jest.fn().mockImplementation((groupID) => {
+        const group = new Group(groupID, []);
+        groups[`group_${groupID}`] = group
+        return group;
+    })
+};
+
+const mockConstructor = jest.fn().mockImplementation(() => mock);
+
+jest.mock('zigbee-herdsman', () => ({
+    Controller: mockConstructor,
+    Zcl: {ManufacturerCode: {Philips: 4107}},
+}));
+
+module.exports = {
+    events, ...mock, constructor: mockConstructor, devices, groups, returnDevices
+};

test/utils.js

@@ -1,14 +0,0 @@
-const logger = require('../lib/util/logger');
-
-module.exports = {
-    stubLogger: (jest) => {
-        jest.spyOn(logger, 'info').mockReturnValue(undefined);
-        jest.spyOn(logger, 'warn').mockReturnValue(undefined);
-        jest.spyOn(logger, 'debug').mockReturnValue(undefined);
-        jest.spyOn(logger, 'error').mockReturnValue(undefined);
-        jest.spyOn(logger, 'trace').mockReturnValue(undefined);
-    },
-    zigbeeMessage: (device, cid, type, data, epId, groupid=0) => {
-        return {data: {cid, data}, type, groupid, endpoints: [{device, epId}]};
-    },
-};

test/utils.test.js

@@ -1,78 +1,58 @@
 const utils = require('../lib/util/utils.js');
-const testUtils = require('./utils');
+const version = require('../package.json').version;
 
 describe('Utils', () => {
-    beforeAll(() => {
-        testUtils.stubLogger(jest);
-    });
-
     describe('Is xiaomi device', () => {
         it('Identify xiaomi device', () => {
-            const device = {type: 'Router', manufId: 4151, manufName: 'Xiaomi'};
+            const device = {type: 'Router', manufacturerID: 4151, manufacturerName: 'Xiaomi'};
             expect(true).toBe(utils.isXiaomiDevice(device));
         });
 
-        it('Identify xiaomi device without manufName', () => {
-            const device = {type: 'Router', manufId: 4447};
+        it('Identify xiaomi device without manufacturerName', () => {
+            const device = {type: 'Router', manufacturerID: 4447};
             expect(true).toBe(utils.isXiaomiDevice(device));
         });
 
-        it('Identify xiaomi device with different manufName', () => {
-            const device = {type: 'Router', manufId: 4151, manufName: 'Trust International B.V.\u0000'};
+        it('Identify xiaomi device with different manufacturerName', () => {
+            const device = {type: 'Router', manufacturerID: 4151, manufacturerName: 'Trust International B.V.\u0000'};
             expect(false).toBe(utils.isXiaomiDevice(device));
         });
-
-        it('Identify QBKG03LM as enddevice', () => {
-            const device = {type: 'Router', manufId: 4447, modelId: 'lumi.ctrl_neutral1'};
-            expect(false).toBe(utils.isRouter(device));
-            expect('EndDevice').toBe(utils.correctDeviceType(device));
-        });
-
-        it('Identify QBKG04LM as enddevice', () => {
-            const device = {type: 'Router', manufId: 4447, modelId: 'lumi.ctrl_neutral2'};
-            expect(false).toBe(utils.isRouter(device));
-            expect('EndDevice').toBe(utils.correctDeviceType(device));
-        });
     });
 
-    describe('Get endpoint by id', () => {
-        it('Pick default ep', () => {
-            const zigbee = {
-                getDevice: (entityID) => {
-                    return {modelId: 'TRADFRI on/off switch'};
-                },
-                getEndpoint: (entityID, epId) => {
-                    return {epId: epId == null ? 1 : 0};
-                },
-            };
-            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
-            expect(endpoint.epId).toBe(1);
-        });
+    it('Convert milliseconds to seconds', () => {
+        expect(utils.millisecondsToSeconds(2000)).toBe(2);
+    })
 
-        it('Pick default ep from mapping when default defined', () => {
-            const zigbee = {
-                getDevice: (entityID) => {
-                    return {modelId: 'SML002'};
-                },
-                getEndpoint: (entityID, epId) => {
-                    return {epId};
-                },
-            };
-            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
-            expect(endpoint.epId).toBe(2);
-        });
+    it('Object has properties', () => {
+        expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b'])).toBeTruthy();
+        expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b', 'd'])).toBeFalsy();
+    })
 
-        it('Pick default ep from mapping when not defined', () => {
-            const zigbee = {
-                getDevice: (entityID) => {
-                    return {modelId: 'lumi.sensor_86sw2.es1'};
-                },
-                getEndpoint: (entityID, epId) => {
-                    return {epId: epId == null ? 1 : 0};
-                },
-            };
-            const endpoint = utils.getEndpointByEntityID(zigbee, '0x12345678', null);
-            expect(endpoint.epId).toBe(1);
-        });
-    });
+    it('git last commit', async () => {
+        let mockReturnValue = [];
+        jest.mock('git-last-commit', () => ({
+            getLastCommit: (cb) => cb(mockReturnValue[0], mockReturnValue[1])
+        }));
+
+        mockReturnValue = [false, {shortHash: '123'}]
+        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "123", "version": version});
+
+        mockReturnValue = [true, null]
+        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
+    })
+
+    it('To local iso string', async () => {
+        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
+        var getTimezoneOffset = Date.prototype.getTimezoneOffset;
+        Date.prototype.getTimezoneOffset = () => 60;
+        expect(utils.formatDate(date, 'ISO_8601_local').endsWith('-01:00')).toBeTruthy();
+        Date.prototype.getTimezoneOffset = () => -60;
+        expect(utils.formatDate(date, 'ISO_8601_local').endsWith('+01:00')).toBeTruthy();
+        Date.prototype.getTimezoneOffset = getTimezoneOffset;
+    })
+
+    it('Throw exception when formating with invalid date', async () => {
+        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
+        expect(() => utils.formatDate(date, 'invalid', 1)).toThrowError("Unsupported type 'invalid'")
+    })
 });

update.sh

@@ -11,8 +11,7 @@ git checkout HEAD -- npm-shrinkwrap.json
 git pull
 
 echo "Installing dependencies..."
-rm -rf node_modules
-npm install
+npm ci
 
 echo "Restore configuration..."
 cp -R data-backup/* data