1.12.1

* Home Assistant support for ZG2819S/511.344

* Added 'homeassistant_legacy_triggers' support for devices with multiple endpoints

* Changed code style to pass lint test

* Reverted fix for the action and click trigger to send an empty message for devices with multiple endpoints

* Home Assistant support for the updated ZG2819S converter

* Update homeassistant.js

Co-authored-by: Koen Kanters <koenkanters94@gmail.com>

.dockerignore

@@ -1,7 +1,15 @@
-node_modules
-.travis*
+.eslintignore
+.eslintrc.json
 .git
+.github
 .gitignore
-*.png
-firmware
-docker/Dockerfile*
+.npmrc
+.travis*
+.travis.yml
+README.md
+docker/Dockerfile*
+images
+node_modules
+scripts
+test
+update.sh

.eslintignore

@@ -1 +1,2 @@
-node_modules/*
+node_modules/*
+test/

.eslintrc.json

@@ -12,7 +12,8 @@
     "rules": {
         "require-jsdoc": "off",
         "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }]
+        "max-len": ["error", { "code": 120 }],
+        "no-prototype-builtins": "off"
     },
     "plugins": [
         "jest"

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [koenkk]
+custom: https://www.paypal.me/koenkk

.github/branch-switcher.yml

@@ -0,0 +1 @@
+preferredBranch: dev

.gitignore

@@ -63,4 +63,10 @@ data/config.json
 data/log*.txt
 data/state.json
 data/log
-data-backup/
+data-backup/
+data/coordinator_backup.json
+data/.storage
+data/database.db.backup
+
+# MacOS indexing file
+.DS_Store

.npmignore

@@ -0,0 +1,25 @@
+#tests
+test
+coverage
+
+#build tools
+.travis.yml
+.jenkins.yml
+.codeclimate.yml
+.github
+azure-pipelines.yaml
+
+#linters
+.jscsrc
+.jshintrc
+.eslintrc*
+.dockerignore
+.eslintignore
+
+#editor settings
+.vscode
+
+#src
+docs
+docker
+images

.travis.yml

@@ -1,33 +0,0 @@
-language: node_js
-
-sudo: required
-
-services:
-  - docker
-
-node_js:
-  - "10"
-
-before_install:
-  - "echo '{\"experimental\": true}' | sudo tee /etc/docker/daemon.json"
-  - sudo apt update -y
-  - sudo apt install --only-upgrade docker-ce -y
-  - sudo service docker restart
-
-install:
-  - npm install
-
-before_script:
-  - npm test
-  - npm run eslint
-
-script:
-  - ".travis/docker.sh"
-  - ".travis/trigger-downstream.sh"
-
-env:
-  global:
-    - DOCKER_USERNAME=koenkk
-    - secure: q5W9+mHz13PslLkTxuYNXivOau+N5OyB53sqgb+krOCPlbN9+gvi2Av9SribVd5jp2yx2O9rV/mr8eS0M2ykvBJ3UsIsCXiFL1dEPmsz1wlFAnW3kbD2EGhGFEV0dWRflSlA+5iQtcMPvbYu2PAOChUUoY7NNlhcz2EC+XXUapvvvaDwA1oSdtX+m61qyREshZb8B57DyXhziy6i/xjhFa84C0mR/iGFRWpZgcxmrNFH+/XDhSIpdq2KshhDWhjyag3ukAsTxTpqzLh1rFRlczjJEn0CcHG4WHsWHUFn9KK2lvT/YCDP8w0URnR1bCr8nyxGZJ2DLW6AzxZtJfWjvbeZYjO2R3WeHpzp6Jw6Qz5pG3gmu2FgeFpZ3FTEGtJ/Ev0/pjkY2Msx9mItzUJ7D8hY2noQZMPASar6oXEDYlkomx7gHHla0MPv3PKSJg5LX6Ybl137GIa6MLYYyz6SEKfyRUQhy3Itqrb/ID/ZP0UuLgyyDfShb8A6XOnsVqwZAsOPtU2UPnD1vsVXhjkpflK9+7QjL5m9MSqfAhg6qtwNsswl1KSiyhmdldGhZW5OfUasDxZKHyx7bsS1BhNnLi/iy4ifDh6KSr64aUW4iM0X99gkU+662Hgf73QpIG+oP41r30iEN0unTcRrFQDIbTSSapbA95bwUCBF1CgQC20=
-    - secure: em3lBVXvZ3kOVI4ipVlh93NIU6guLvyzHMmsLDuxhc92VFURqlYu3iXTTNEnknwyE31mziAkVFnmj+3OENd/VfzP01d4KKaYAQrbn2AnOMvcIIExLfaFU5QqbW1RMwRBIifhvrmboKt07pFsUy3Rghs80SwNSFDfLFoMhFMz+NdIZfJGW8LJy+gUPmz10YGzqh/chreowqOemgBBoswCGwIh1xB1KNejew3bQllnwQYGXcGuTDwDT4Wz+ObxlFt7YR/4n4oJY8jQYRPolM9qHQAGyn/Cn0TL3c8JqiBpVc1hm2OhYSDqvK0xf5IKsD+kIfFJPGa7nF2cwmhKdUNYR47oue3KB5tdrtDhwqUooTglvBOSXUIbthUR9Xu40ziyd5tHy0ixaLnDRGuKIIvEnXlcE6CgApoe3y2kHYa5dY+eUKxWwvH7b5hG+u82dYvw3VYpCHAnDQVgUzz4F63qZ2WZhcH0W3PF3/ytVAHyMu2vAMSrSXGi5xgM6el7Gp7SFVTFXBurbOaoJ3ZwwSEDP5d8mDyk1d8thmkQFL9P6iWtNnXOzjv4GIliEDWQg76BX6MLD3n3fnP0aq+RVmv+ev2Lp4xZZDfgGPek7g35IsaiohH6YvHloxE+ETQ0Ttb1lCQos9fj7LlI+ud6ExjwyRuEnzCOzr/qC/3Leg85Wc8=
-    - secure: GbJVFFY4nmQovp8i2VoQ1AOV8h9QoAcleBV9Mg5TItA8U2Y+6t9wx0iWbaclxOrzI727XZOk5HquMTKKTI430lT6OjU01+YJqwyq+k650YiX5PImsgJXGhFzBKeUcB3A53kKijfr9aEV00YiXRsKoKAn+Bl09m/9O+melBbTwO/YAQyt6EoUt21Yk0BoqpW2CTMPAgsU5tm6Ra3bbJ+IaUYVMYBtuHqtGiKg8jgTHqRhwKejDyV7+1dDxX8q51bK2Kb6PujbI1SS5wsJEpkU/eoPs7MuprmlYarYtBt1fDXSK1EOjcfpyEFRlrNLewIKiFEo3dkTOEtW2a4t3tcAiXOj8Hcvp+6RsjTfnjYmZweHVh/JKDhujjHBMviyTjwp4RnPaQSFou0iVIB5+Mo34hUhFDXc30SBzaux+lyR3rGoEu5pubZNjG5CkKIpw3IyduNKtIrlbCnfg+WDU3jrXmk0fzmwyriyXhx9BAzM+hyFiElZ+3xoXAlmDlwvaRZyJuQQNslwGPbZyX9EByIUkECjYPf92QPyD0mjbn8C32IphgQ515Up+15g2meDniugzowLJkDe3W86M5fB1kPQiDq5oT4M6TgAPRElLN6/ibyfOzDAtOdZgUiQvFnUQb1s9LunZqrc57AKzuzl2xG9Ok/cUkZFSOjrFr40jXgS2Lw=

.travis/docker.sh

@@ -1,68 +0,0 @@
-#!/bin/bash -e
-
-login() {
-  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-}
-
-tag_push() {
-  docker tag $DOCKER_USERNAME/zigbee2mqtt:$1 $DOCKER_USERNAME/zigbee2mqtt:$2
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$2
-}
-
-build_and_push() {
-  docker build --platform=$3 --build-arg COMMIT=$(git rev-parse --short HEAD) -t $DOCKER_USERNAME/zigbee2mqtt:$1 -f $2 .
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$1
-}
-
-push_manifest() {
-  export DOCKER_CLI_EXPERIMENTAL=enabled  
-
-  docker manifest create $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1
-
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm32v6$1 --os linux --arch arm --variant v6
-  docker manifest annotate $DOCKER_USERNAME/zigbee2mqtt:latest$1 $DOCKER_USERNAME/zigbee2mqtt:arm64v8$1 --os linux --arch arm64 --variant v8
-
-  docker manifest inspect $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker manifest push -p $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  docker run --rm mplatform/mquery $DOCKER_USERNAME/zigbee2mqtt:latest$1
-
-  export DOCKER_CLI_EXPERIMENTAL=disabled
-}
-
-# Only update Docker images for:
-# - dev branch
-# - version release
-if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ ! -z ${TRAVIS_TAG+x} ] && [ ! "$TRAVIS_TAG" = "" ]
-then
-  echo "Updating docker images for master branch!"
-  login
-
-  # Push versioned images
-  build_and_push "$TRAVIS_TAG" docker/Dockerfile.amd64 amd64
-  build_and_push "$TRAVIS_TAG-arm32v6" docker/Dockerfile.arm32v6 arm
-  build_and_push "$TRAVIS_TAG-arm64v8" docker/Dockerfile.arm64v8 arm64
-
-  # Push latest images.
-  tag_push "$TRAVIS_TAG-arm32v6" "arm32v6"
-  tag_push "$TRAVIS_TAG-arm64v8" "arm64v8"
-  tag_push "$TRAVIS_TAG" latest
-
-  # Push manifest
-  push_manifest ""
-
-elif [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Updating docker images for dev branch!"
-  login
-  build_and_push latest-dev docker/Dockerfile.amd64 amd64
-  build_and_push arm32v6-dev docker/Dockerfile.arm32v6 arm
-  build_and_push arm64v8-dev docker/Dockerfile.arm64v8 arm64
-
-  # Push manifest
-  push_manifest "-dev"
-
-else
-  echo "Not updating docker images, triggered by pull request or not on master/dev branch"
-fi

.travis/trigger-downstream.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-function travis_trigger() {
-     local org=$1 && shift
-     local repo=$1 && shift
-     local branch=${1:-master} && shift
-
-     body="{
-             \"request\": {
-               \"branch\": \"${branch}\"
-              }
-           }"
-
-     curl -s -X POST \
-          -H "Content-Type: application/json" \
-          -H "Accept: application/json" \
-          -H "Travis-API-Version: 3" \
-          -H "Authorization: token $TRAVIS_TOKEN" \
-          -d "$body" \
-          "https://api.travis-ci.org/repo/${org}%2F${repo}/requests"
-}
-
-# Only trigger downstream if on dev branch and not pull request
-if [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Triggering build of downstream projects!"
-  travis_trigger "danielwelch" "hassio-zigbee2mqtt" "master"
-else
-  echo "Not triggering build of downstream projects, triggered by pull request or not on dev branch"
-fi

README.md

@@ -5,8 +5,8 @@
     <br>
     <br>
     <div style="display: flex;">
-        <a href="https://travis-ci.org/Koenkk/zigbee2mqtt/">
-            <img src="https://img.shields.io/travis/Koenkk/zigbee2mqtt.svg">
+        <a href="https://dev.azure.com/koenkk/zigbee/_build?definitionId=5">
+            <img src="https://dev.azure.com/koenkk/zigbee/_apis/build/status/Koenkk.zigbee2mqtt?branchName=dev">
         </a>
         <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
             <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
@@ -23,6 +23,15 @@
         <a href="http://zigbee2mqtt.discourse.group/">
             <img src="https://img.shields.io/discourse/https/zigbee2mqtt.discourse.group/status.svg">
         </a>
+        <a>
+            <img src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg">
+        </a>
+        <a href="https://www.codacy.com/manual/Koenkk/zigbee2mqtt?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=Koenkk/zigbee2mqtt&amp;utm_campaign=Badge_Grade">
+            <img src="https://api.codacy.com/project/badge/Grade/24f1e0fe39f04daa810e8a1416693d3f">
+        </a>
+        <a href="https://www.npmjs.com/package/zigbee2mqtt">
+            <img src="https://img.shields.io/npm/v/zigbee2mqtt">
+        </a>
     </div>
     <h1>Zigbee2mqtt  🌉 🐝</h1>
     <p>

azure-pipelines.yaml

@@ -0,0 +1,90 @@
+trigger:
+  tags:
+    include:
+    - '*'
+  branches:
+    include:
+    - '*'
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+steps:
+
+# Prepare
+- task: NodeTool@0
+  inputs:
+    versionSpec: '12.x'
+  displayName: 'Install Node.js'
+- bash: npm ci
+  displayName: 'Install dependencies'
+
+# Verify
+- bash: npm run test-with-coverage
+  displayName: 'Test'
+- bash: npm run eslint
+  displayName: 'Lint'
+
+# NPM
+- task: Npm@1
+  inputs:
+    command: publish
+    publishEndpoint: npm
+  displayName: 'Publish to npm'
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
+
+# Docker
+- task: DockerInstaller@0
+  displayName: Docker install
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  inputs:
+    dockerVersion: 19.03.5
+    releaseType: stable
+- task: Docker@2
+  displayName: Docker registry login
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  inputs:
+    command: login
+    containerRegistry: docker_registry
+- bash: |
+    sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
+    sudo chmod a+x /usr/local/bin/buildx
+    docker run --rm --privileged hypriot/qemu-register:v2.7.0
+    buildx create --use
+    buildx ls
+  displayName: 'Docker setup'
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+- bash: |
+    buildx build \
+      --build-arg COMMIT=$(git rev-parse --short HEAD) \
+      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+      -f docker/Dockerfile \
+      --push \
+      -t koenkk/zigbee2mqtt:latest-dev \
+      .
+  displayName: 'Docker build dev'
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))
+- bash: |
+    TAG="$(git describe --tags)"
+    buildx build \
+      --build-arg COMMIT=$(git rev-parse --short HEAD) \
+      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+      -f docker/Dockerfile \
+      --push \
+      -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
+      .
+  displayName: 'Docker build release'
+  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
+
+# Trigger downstream
+- bash: |
+    BODY="{\"request\": {\"branch\": \"master\"}}"
+    curl -s -X POST \
+        -H "Content-Type: application/json" \
+        -H "Accept: application/json" \
+        -H "Travis-API-Version: 3" \
+        -H "Authorization: token $(TRAVIS_TOKEN)" \
+        -d "$BODY" \
+        "https://api.travis-ci.org/repo/danielwelch%2Fhassio-zigbee2mqtt/requests"
+  displayName: 'Trigger Hass.io build'
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

docker/Dockerfile

@@ -0,0 +1,30 @@
+FROM node:12-alpine as base
+
+WORKDIR /app
+RUN apk add --no-cache tzdata eudev
+
+COPY package.json .
+
+# Dependencies
+FROM base as dependencies
+
+COPY npm-shrinkwrap.json .
+
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
+    npm ci --production && \
+    apk del .buildtools
+
+# Release
+FROM base as release
+
+COPY --from=dependencies /app/node_modules ./node_modules
+COPY lib ./lib
+COPY index.js docker/run.sh data/configuration.yaml ./
+
+RUN chmod +x /app/run.sh
+RUN mkdir /app/data
+
+ARG COMMIT
+RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+
+ENTRYPOINT ["./run.sh"]

docker/Dockerfile.amd64

@@ -1,20 +0,0 @@
-FROM alpine:3.9
-
-# Copy files
-ADD . /app
-RUN cp /app/data/configuration.yaml /app
-RUN cp /app/docker/run.sh /app
-RUN chmod +x /app/run.sh
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
-
-# Install dependencies
-RUN apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-    npm install --unsafe-perm && \
-    apk del make gcc g++ python linux-headers udev git
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm32v6

@@ -1,22 +0,0 @@
-FROM arm32v6/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-arm-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-arm-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-arm-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm64v8

@@ -1,22 +0,0 @@
-FROM arm64v8/alpine:3.9
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-aarch64-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs npm git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/run.sh

@@ -13,8 +13,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-if [ ! -z "$DEBUG" ]; then
-    npm start
-else
-    DEBUG="$DEBUG" npm start
-fi
+exec npm start

docs/_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-cayman

docs/index.md

@@ -1 +0,0 @@
-Moved to [https://www.zigbee2mqtt.io](https://www.zigbee2mqtt.io)

images/logo_vector.svg

@@ -0,0 +1,146 @@
+<svg class="thumb" width="1000" height="1000" viewBox="0,0,600,600" xmlns="http://www.w3.org/2000/svg" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ns1="http://www.inkscape.org/namespaces/inkscape" xmlns:sketch="http://www.bohemiancoding.com/sketch/ns">
+  <defs></defs>
+  <g transform="matrix(1.8288,0,0,1.8288,-76.3478,13.5303)">
+    <g transform="matrix(1.6652,0,0,1.6704,50.2843,0.6537)">
+      <svg width="186.77" height="186.769" viewBox="-5,-5,186.77,186.769">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk4">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g>
+          <polygon points="51.774,176.769 0,124.995 0,51.774 51.774,0 124.995,0 176.77,51.774 176.77,124.995   124.995,176.769 " fill="url('#linearGradientSk1zes2vk4')"></polygon>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6204,0,0,1.6204,53.946,5.3)">
+      <svg width="186.77" height="186.769" viewBox="-5,-5,186.77,186.769">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vka">
+            <stop offset="0%" stop-color="#ffc038"></stop>
+            <stop offset="100%" stop-color="#ffc038"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g>
+          <polygon points="51.774,176.769 0,124.995 0,51.774 51.774,0 124.995,0 176.77,51.774 176.77,124.995   124.995,176.769 " fill="url('#linearGradientSk1zes2vka')"></polygon>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,185.8484,123.0829)">
+      <svg width="27.265" height="16.606891038197716" viewBox="36.308,71.828,27.265,16.606891038197716">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vkg">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M41.308,76.828c1.197,2.07,2.931,3.953,5.371,5.497c0,0,1.453,1.163,3.373,1.108c1.918,0.051,3.375-1.107,3.375-1.107   c2.289-1.446,3.958-3.192,5.146-5.112c-2.274,0.496-4.875,0.754-7.556,0.754C47.714,77.968,44.316,77.583,41.308,76.828z" fill="url('#linearGradientSk1zes2vkg')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,182.6191,109.0829)">
+      <svg width="31.991999999999997" height="16.800092191011956" viewBox="34.051,63.36,31.991999999999997,16.800092191011956">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vkm">
+            <stop offset="0%" stop-color="#ffffff"></stop>
+            <stop offset="100%" stop-color="#ffffff"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M39.051,68.36c0.055,1.736,0.307,3.47,0.818,5.141c5.94,2.063,15.1,2.206,20.244,0.407c0.563-1.692,0.85-3.453,0.93-5.221   c-2.995,0.938-6.551,1.364-10.114,1.364C46.659,70.052,42.394,69.434,39.051,68.36z" fill="url('#linearGradientSk1zes2vkm')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,182.6191,96.0829)">
+      <svg width="31.867999999999995" height="16.630912092367097" viewBox="34.128,55.641000000000005,31.867999999999995,16.630912092367097">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vks">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M50.929,62.06c-3.902,0-7.81-0.511-11.006-1.419c-0.373,1.537-0.658,3.156-0.795,4.812   c5.78,2.114,15.619,2.598,21.868,0.309c-0.119-1.662-0.384-3.297-0.745-4.846C57.427,61.692,54.187,62.06,50.929,62.06z" fill="url('#linearGradientSk1zes2vks')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,185.041,84.8902)">
+      <svg width="28.836000000000006" height="15.788046460420631" viewBox="35.678,48.491,28.836000000000006,15.788046460420631">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vky">
+            <stop offset="0%" stop-color="#ffffff"></stop>
+            <stop offset="100%" stop-color="#ffffff"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M59.514,58.241c-0.571-1.815-1.232-3.435-1.862-4.75c-2.599,0.543-5.42,0.809-8.212,0.809   c-2.458,0-4.881-0.204-7.097-0.579c-0.564,1.205-1.146,2.646-1.665,4.246C46.062,59.501,53.848,59.813,59.514,58.241z" fill="url('#linearGradientSk1zes2vky')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(1.6146,0,0,1.6146,120.4554,23.9828)">
+      <svg width="108.32981691822947" height="57.54144295152284" viewBox="-4.359346882061053,10.889,108.32981691822947,57.54144295152284">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk14">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g></g>
+        <g>
+          <path d="M58.786,38.047c-0.651-1.481-1.406-2.589-1.406-2.589c-0.11-0.123-0.253-0.232-0.394-0.342  c0.422-1.017,0.659-2.14,0.659-3.324c0-3.374-1.896-6.271-4.6-7.523l2.116-7.369l6.143-0.228c0.215-0.007,0.382-0.188,0.378-0.405  c-0.008-0.215-0.19-0.375-0.406-0.378l-6.396,0.238c-0.152,0.006-0.287,0.095-0.346,0.239l-3.563,7.305  c-0.302-0.039-0.607-0.065-0.92-0.065c-0.314,0-0.621,0.026-0.924,0.066l-3.565-7.308c-0.058-0.138-0.193-0.23-0.349-0.237  l-6.393-0.236c-0.216-0.001-0.396,0.161-0.407,0.376c-0.006,0.218,0.163,0.398,0.38,0.405l6.143,0.23l2.118,7.368  c-2.703,1.252-4.598,4.147-4.597,7.52c0,1.184,0.239,2.307,0.659,3.323c-0.148,0.11-0.288,0.223-0.393,0.344  c0,0-0.739,1.108-1.38,2.663C29.184,41.662-2.639,46.12,0.916,55.936c3.944,10.877,25.455,7.546,29.052,4.598  c5.007-4.104,9.231-7.95,11.843-11c0.224,0.512,0.459,0.979,0.694,1.396c4.583,0.822,10.289,0.82,14.998-0.245  c0.166-0.346,0.332-0.731,0.496-1.137c2.625,3.011,6.768,6.769,11.648,10.77c3.596,2.949,25.106,6.275,29.048-4.599  C102.202,46.038,71.295,41.566,58.786,38.047z" fill="url('#linearGradientSk1zes2vk14')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(0.5515,-0.0096,0.0096,0.5515,184.0296,161.5473)">
+      <svg width="91.9" height="160" style="overflow: visible;">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk1g">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g transform="matrix(1,0,0,1,0,75)">
+          <path d="M 28.2-77.4L 28.2-77.4L 28.2 5.4Q 28.2 7.3  27.35 8.15Q 26.5 9  24.6 9L 24.6 9L 10.1 9Q 8.2 9  7.35 8.15Q 6.5 7.3  6.5 5.4L 6.5 5.4L 6.5-77.4Q 6.5-79.3  7.35-80.15Q 8.2-81  10.1-81L 10.1-81L 24.6-81Q 26.5-81  27.35-80.15Q 28.2-79.3  28.2-77.4ZM 85.39999999999998-77.4L 85.39999999999998-77.4L 85.39999999999998 5.4Q 85.39999999999998 7.3  84.54999999999998 8.15Q 83.69999999999999 9  81.79999999999998 9L 81.79999999999998 9L 67.29999999999998 9Q 65.39999999999998 9  64.54999999999998 8.15Q 63.69999999999998 7.3  63.69999999999998 5.4L 63.69999999999998 5.4L 63.69999999999998-77.4Q 63.69999999999998-79.3  64.54999999999998-80.15Q 65.39999999999998-81  67.29999999999998-81L 67.29999999999998-81L 81.79999999999998-81Q 83.69999999999999-81  84.54999999999998-80.15Q 85.39999999999998-79.3  85.39999999999998-77.4Z" data-textpath="| |" data-text-anchor="middle" data-font-size="100" data-letterspacing="0" data-side="left" data-offset="0.5" data-dx="0" data-dy="0" data-xmlspace="preserve" data-href="" data-font="Bungee" data-font-src="/Content/clientfonts/bungee.woff" data-font-scale="contain" transform="translate(0,0)" data-x="0" data-y="0" fill="url('#linearGradientSk1zes2vk1g')"></path>
+        </g>
+      </svg>
+    </g>
+    <g transform="matrix(0.5,0,0,0.5,136.759,231)">
+      <svg width="289.30000000000007" height="160" style="overflow: visible;">
+        <defs>
+          <linearGradient x1="0" y1="0.5" x2="1" y2="0.5" id="linearGradientSk1zes2vk1j">
+            <stop offset="0%" stop-color="#000000"></stop>
+            <stop offset="100%" stop-color="#000000"></stop>
+          </linearGradient>
+        </defs>
+        <g transform="matrix(1,0,0,1,0,75)">
+          <path d="M 25.4 0L 25.4 0L 10.5 0Q 8.6 0  7.75-0.85Q 6.9-1.7  6.9-3.6L 6.9-3.6L 6.9-68.4Q 6.9-70.3  7.75-71.15Q 8.6-72  10.5-72L 10.5-72L 26.6-72Q 28.2-72  29.25-71.3Q 30.3-70.6  31.2-69L 31.2-69L 40.2-52.7Q 40.7-51.8  41.05-51.4Q 41.4-51  42-51L 42-51L 42.9-51Q 43.5-51  43.85-51.4Q 44.2-51.8  44.7-52.7L 44.7-52.7L 53.6-69Q 54.5-70.6  55.55-71.3Q 56.6-72  58.2-72L 58.2-72L 74.4-72Q 76.3-72  77.15-71.15Q 78-70.3  78-68.4L 78-68.4L 78-3.6Q 78-1.7  77.15-0.85Q 76.3 0  74.4 0L 74.4 0L 59 0Q 57.1 0  56.25-0.85Q 55.4-1.7  55.4-3.6L 55.4-3.6L 55.4-38.8L 49.5-27.9Q 48.6-26.2  47.35-25.45Q 46.1-24.7  44.2-24.7L 44.2-24.7L 40.2-24.7Q 38.3-24.7  37.05-25.45Q 35.8-26.2  34.9-27.9L 34.9-27.9L 29-38.8L 29-3.6Q 29-1.7  28.15-0.85Q 27.3 0  25.4 0ZM 129.4 9L 129.4 9L 114 9Q 112.10000000000001 9  111.25 8.15Q 110.4 7.3  110.4 5.4L 110.4 5.4L 110.4-0.1Q 98.80000000000001-1.8  94.75-7.05Q 90.7-12.3  90.7-19.8L 90.7-19.8L 90.7-53Q 90.7-57.6  92.05000000000001-61.35Q 93.4-65.1  96.85000000000001-67.8Q 100.30000000000001-70.5  106.4-72Q 112.5-73.5  121.80000000000001-73.5L 121.80000000000001-73.5Q 131.10000000000002-73.5  137.15-72Q 143.2-70.5  146.65-67.8Q 150.10000000000002-65.1  151.45-61.35Q 152.8-57.6  152.8-53L 152.8-53L 152.8-19.8Q 152.8-16  151.9-12.75Q 151-9.5  148.8-7Q 146.60000000000002-4.5  142.75-2.7Q 138.9-0.9  133-0.1L 133-0.1L 133 5.4Q 133 7.3  132.15 8.15Q 131.3 9  129.4 9ZM 113.4-49.3L 113.4-22Q 113.4-20  114.9-18.65Q 116.4-17.3  121.80000000000001-17.3L 121.80000000000001-17.3Q 127.2-17.3  128.7-18.65Q 130.2-20  130.2-22L 130.2-22L 130.2-49.3Q 130.2-51.2  128.7-52.55Q 127.2-53.9  121.80000000000001-53.9L 121.80000000000001-53.9Q 116.4-53.9  114.9-52.55Q 113.4-51.2  113.4-49.3L 113.4-49.3ZM 199 0L 199 0L 182.79999999999998 0Q 180.9 0  180.04999999999998-0.85Q 179.2-1.7  179.2-3.6L 179.2-3.6L 179.2-52.7L 164.5-52.7Q 162.6-52.7  161.75-53.55Q 160.9-54.4  160.9-56.3L 160.9-56.3L 160.9-68.4Q 160.9-70.3  161.75-71.15Q 162.6-72  164.5-72L 164.5-72L 217.3-72Q 219.2-72  220.05-71.15Q 220.89999999999998-70.3  220.89999999999998-68.4L 220.89999999999998-68.4L 220.89999999999998-56.3Q 220.89999999999998-54.4  220.05-53.55Q 219.2-52.7  217.3-52.7L 217.3-52.7L 202.6-52.7L 202.6-3.6Q 202.6-1.7  201.75-0.85Q 200.89999999999998 0  199 0ZM 264.6 0L 264.6 0L 248.40000000000003 0Q 246.50000000000006 0  245.65000000000003-0.85Q 244.80000000000004-1.7  244.80000000000004-3.6L 244.80000000000004-3.6L 244.80000000000004-52.7L 230.10000000000005-52.7Q 228.20000000000005-52.7  227.35000000000005-53.55Q 226.50000000000006-54.4  226.50000000000006-56.3L 226.50000000000006-56.3L 226.50000000000006-68.4Q 226.50000000000006-70.3  227.35000000000005-71.15Q 228.20000000000005-72  230.10000000000005-72L 230.10000000000005-72L 282.90000000000003-72Q 284.80000000000007-72  285.65000000000003-71.15Q 286.50000000000006-70.3  286.50000000000006-68.4L 286.50000000000006-68.4L 286.50000000000006-56.3Q 286.50000000000006-54.4  285.65000000000003-53.55Q 284.80000000000007-52.7  282.90000000000003-52.7L 282.90000000000003-52.7L 268.20000000000005-52.7L 268.20000000000005-3.6Q 268.20000000000005-1.7  267.35-0.85Q 266.50000000000006 0  264.6 0Z" data-textpath="MQTT" data-text-anchor="middle" data-font-size="100" data-letterspacing="0" data-side="left" data-offset="0.5" data-dx="0" data-dy="0" data-xmlspace="preserve" data-href="" data-font="Bungee" data-font-src="/Content/clientfonts/bungee.woff" data-font-scale="contain" transform="translate(0,0)" data-x="0" data-y="0" fill="url('#linearGradientSk1zes2vk1j')"></path>
+        </g>
+      </svg>
+    </g>
+  </g>
+</svg>

index.js

@@ -13,6 +13,11 @@ controller.start();
 process.on('SIGINT', handleQuit);
 process.on('SIGTERM', handleQuit);
 
+let stopping = false;
+
 function handleQuit() {
-    controller.stop(() => process.exit());
+    if (!stopping) {
+        stopping = true;
+        controller.stop();
+    }
 }

lib/controller.js

@@ -1,104 +1,120 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
 const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
 
 // Extensions
+const ExtensionEntityPublish = require('./extension/entityPublish');
+const ExtensionDeviceReceive = require('./extension/deviceReceive');
 const ExtensionNetworkMap = require('./extension/networkMap');
 const ExtensionSoftReset = require('./extension/softReset');
-const ExtensionXiaomi = require('./extension/xiaomi');
-const ExtensionDevicePublish = require('./extension/devicePublish');
 const ExtensionHomeAssistant = require('./extension/homeassistant');
 const ExtensionDeviceConfigure = require('./extension/deviceConfigure');
 const ExtensionDeviceGroupMembership = require('./extension/deviceGroupMembership');
-const ExtensionDeviceReceive = require('./extension/deviceReceive');
-const ExtensionBridgeConfig = require('./extension/bridgeConfig');
+const ExtensionBridgeLegacy = require('./extension/bridgeLegacy');
 const ExtensionGroups = require('./extension/groups');
 const ExtensionDeviceAvailability = require('./extension/deviceAvailability');
 const ExtensionDeviceBind = require('./extension/deviceBind');
 const ExtensionDeviceReport = require('./extension/deviceReport');
-const ExtensionLivolo = require('./extension/livolo');
+const ExtensionDeviceEvent = require('./extension/deviceEvent');
+const ExtensionOTAUpdate = require('./extension/otaUpdate');
 
 class Controller {
     constructor() {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
+        this.eventBus = new EventBus();
+        this.state = new State(this.eventBus);
 
-        if (settings.get().advanced.cache_state) {
-            this.state = new State();
-        }
-
-        // Bind methods
-        this.onMQTTConnected = this.onMQTTConnected.bind(this);
-        this.onZigbeeMessage = this.onZigbeeMessage.bind(this);
-        this.onMQTTMessage = this.onMQTTMessage.bind(this);
         this.publishEntityState = this.publishEntityState.bind(this);
+        this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
 
         // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
         this.extensions = [
-            new ExtensionDeviceReceive(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceGroupMembership(this.zigbee, this.mqtt, this.publishEntityState),
-            new ExtensionDeviceConfigure(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDevicePublish(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionNetworkMap(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionXiaomi(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionBridgeConfig(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionGroups(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-            new ExtensionDeviceBind(this.zigbee, this.mqtt, this.state, this.publishEntityState),
-
+            new ExtensionEntityPublish(...args),
+            new ExtensionDeviceReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionDeviceConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionDeviceBind(...args),
+            new ExtensionDeviceEvent(...args),
+            new ExtensionOTAUpdate(...args),
         ];
 
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
+
         if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionDeviceReport(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionDeviceReport(...args));
         }
 
         if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
+        /* istanbul ignore next */
         if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionSoftReset(...args));
         }
 
         if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionDeviceAvailability(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
-        }
-
-        if (settings.get().experimental.livolo) {
-            // https://github.com/Koenkk/zigbee2mqtt/issues/592
-            this.extensions.push(new ExtensionLivolo(
-                this.zigbee, this.mqtt, this.state, this.publishEntityState
-            ));
+            this.extensions.push(new ExtensionDeviceAvailability(...args));
         }
     }
 
-    onMQTTConnected() {
-        // Resend all cached states.
-        this.sendAllCachedStates();
+    async start() {
+        const settingsErrors = settings.validate();
+        if (settingsErrors) {
+            logger.error(`Refusing to start, configuration.yaml is not valid, found the following errors:`);
+            for (const error of settingsErrors) {
+                logger.error(`\t - ${error}`);
+            }
+            logger.error(
+                `If you don't know how to solve this, read https://www.zigbee2mqtt.io/configuration/configuration.html`,
+            );
+            process.exit(1);
+        }
 
-        // Call extensions
-        this.extensions.filter((e) => e.onMQTTConnected).forEach((e) => e.onMQTTConnected());
-    }
+        this.state.start();
 
-    onZigbeeStarted() {
-        // Log zigbee clients on startup and configure.
-        const devices = this.zigbee.getAllClients();
+        const info = await utils.getZigbee2mqttVersion();
+        logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
+
+        // Start zigbee
+        try {
+            await this.zigbee.start();
+            this.callExtensionMethod('onZigbeeStarted', []);
+            this.zigbee.on('event', this.onZigbeeEvent.bind(this));
+            this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
+        } catch (error) {
+            logger.error('Failed to start zigbee');
+            logger.error('Exiting...');
+            logger.error(error.stack);
+            process.exit(1);
+        }
+
+        // Log zigbee clients on startup
+        const devices = this.zigbee.getClients();
         logger.info(`Currently ${devices.length} devices are joined:`);
-        devices.forEach((device) => {
-            logger.info(this.getDeviceStartupLogMessage(device));
-        });
+        for (const device of devices) {
+            const entity = this.zigbee.resolveEntity(device);
+            logger.info(
+                (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
+                ` (${entity.device.ieeeAddr}): ` +
+                (entity.definition ?
+                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
+                    'Not supported ') +
+                `(${entity.device.type})`,
+            );
+        }
 
         // Enable zigbee join.
         if (settings.get().permit_join) {
@@ -107,186 +123,227 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        this.zigbee.permitJoin(settings.get().permit_join);
+        await this.zigbee.permitJoin(settings.get().permit_join);
 
-        // Connect to MQTT broker
-        this.mqtt.connect(this.onMQTTMessage, this.onMQTTConnected);
+        // MQTT
+        this.mqtt.on('message', this.onMQTTMessage.bind(this));
+        await this.mqtt.connect();
+
+        // Send all cached states.
+        if (settings.get().advanced.cache_state) {
+            for (const device of this.zigbee.getClients()) {
+                if (this.state.exists(device.ieeeAddr)) {
+                    this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
+                }
+            }
+        }
+
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
 
         // Call extensions
-        this.extensions.filter((e) => e.onZigbeeStarted).forEach((e) => e.onZigbeeStarted());
+        await this.callExtensionMethod('onMQTTConnected', []);
     }
 
-    onZigbeeMessage(message) {
-        // Variables
-        let device = null;
-        let mappedDevice = null;
+    async stop() {
+        // Call extensions
+        await this.callExtensionMethod('stop', []);
 
-        // Check if message has a device
-        if (message.endpoints && message.endpoints.length && message.endpoints[0].device) {
-            device = message.endpoints[0].device;
+        // Wrap-up
+        this.state.stop();
+        await this.mqtt.disconnect();
+
+        try {
+            await this.zigbee.stop();
+            process.exit(0);
+        } catch (error) {
+            logger.error('Failed to stop zigbee');
+            process.exit(1);
         }
-
-        // Retrieve modelId from message
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        }
-
-        // Log
-        logger.debug(
-            `Received zigbee message of type '${message.type}' with data '${JSON.stringify(message.data)}'` +
-            (device ? ` of device '${device.modelId}' (${device.ieeeAddr})` : '') +
-            (message.groupid ? ` with groupID ${message.groupID}` : '') +
-            (message.endpoints && message.endpoints.length ? ` of endpoint ${message.endpoints[0].epId}` : '')
-        );
-
-        // Call extensions.
-        this.extensions
-            .filter((e) => e.onZigbeeMessage)
-            .forEach((e) => e.onZigbeeMessage(message, device, mappedDevice));
     }
 
-    onMQTTMessage(topic, message) {
+    async onZigbeeAdapterDisconnected() {
+        logger.error('Adapter disconnected, stopping');
+        await this.stop();
+    }
+
+    async onZigbeeEvent(type, data) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && !resolvedEntity.settings) {
+            // Only deviceLeave doesn't have a device (not interesting to add to settings)
+            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
+        }
+
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
+
+        if (type === 'message') {
+            logger.debug(
+                `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
+                `, data '${JSON.stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
+            );
+        } else if (type === 'deviceJoined') {
+            logger.info(`Device '${name}' joined`);
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
+
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
+                } else {
+                    logger.warn(
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
+                    );
+                }
+            } else if (data.status === 'failed') {
+                logger.error(`Failed to interview '${name}', device has not successfully been paired`);
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    logger.info(`Starting interview of '${name}'`);
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            logger.debug(`Device '${name}' announced itself`);
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+            }
+        }
+
+        // Call extensions
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+    }
+
+    onMQTTMessage(payload) {
+        const {topic, message} = payload;
         logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
 
         // Call extensions
-        const results = this.extensions
-            .filter((e) => e.onMQTTMessage)
-            .map((e) => e.onMQTTMessage(topic, message));
-
-        if (!results.includes(true)) {
-            logger.warn(`Cannot handle MQTT message on '${topic}' with data '${message}'`);
-        }
+        this.callExtensionMethod('onMQTTMessage', [topic, message]);
     }
 
-    start() {
-        if (this.state) {
-            this.state.start();
+    async publishEntityState(IDorName, payload, stateChangeReason=null) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
+            logger.error(`'${IDorName}' does not exist, skipping publish`);
+            return;
         }
 
-        this.startupLogVersion(() => {
-            this.zigbee.start(this.onZigbeeMessage, (error) => {
-                if (error) {
-                    logger.error('Failed to start', error);
-                    logger.error('Exiting...');
-                    process.exit(1);
-                } else {
-                    this.onZigbeeStarted();
-                }
-            });
-        });
-    }
+        const isDevice = resolvedEntity.type === 'device';
 
-    stop(callback) {
-        // Call extensions
-        this.extensions.filter((e) => e.stop).forEach((e) => e.stop());
-
-        // Wrap-up
-        if (this.state) {
-            this.state.stop();
+        if (isDevice && settings.get().advanced.last_seen !== 'disable' && resolvedEntity.device.lastSeen) {
+            payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, settings.get().advanced.last_seen);
         }
 
-        this.mqtt.disconnect();
-        this.zigbee.stop(callback);
-    }
-
-    startupLogVersion(callback) {
-        utils.getZigbee2mqttVersion((info) => {
-            logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
-            callback();
-        });
-    }
-
-    getDeviceStartupLogMessage(device) {
-        let friendlyName = 'unknown';
-        let type = 'unknown';
-        let friendlyDevice = {model: 'unkown', description: 'unknown'};
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        if (mappedModel) {
-            friendlyDevice = mappedModel;
-        }
-
-        if (settings.getDevice(device.ieeeAddr)) {
-            friendlyName = settings.getDevice(device.ieeeAddr).friendly_name;
-        }
-
-        if (device.type) {
-            type = device.type;
-        }
-
-        return `${friendlyName} (${device.ieeeAddr}): ${friendlyDevice.model} - ` +
-            `${friendlyDevice.vendor} ${friendlyDevice.description} (${type})`;
-    }
-
-    sendAllCachedStates() {
-        this.zigbee.getAllClients().forEach((device) => {
-            if (this.state && this.state.exists(device.ieeeAddr)) {
-                this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
-            }
-        });
-    }
-
-    publishEntityState(entityID, payload) {
-        const entity = settings.resolveEntity(entityID);
-        const appSettings = settings.get();
         let messagePayload = {...payload};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
+        const newState = objectAssignDeep.noMutate(currentState, payload);
 
-        if (this.state) {
+        // Update state cache with new state.
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
+
+        if (settings.get().advanced.cache_state) {
             // Add cached state to payload
-            if (this.state.exists(entityID)) {
-                messagePayload = objectAssignDeep.noMutate(this.state.get(entityID), payload);
-            }
-
-            // Update state cache with new state.
-            this.state.set(entityID, messagePayload);
+            messagePayload = newState;
         }
 
-        const entitySettings = entity.type === 'device' ? settings.getDevice(entityID) : settings.getGroup(entityID);
+        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: entitySettings ? entitySettings.retain : false,
-            qos: entitySettings && entitySettings.qos ? entitySettings.qos : 0,
+            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
+            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
         };
 
-        if (entity.type === 'device' && appSettings.mqtt.include_device_information) {
-            messagePayload.device = this.getDeviceInfoForMqtt(entityID);
+        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention};
         }
 
-        if (settings.get().experimental.output === 'json') {
-            this.mqtt.publish(entity.friendlyName, JSON.stringify(messagePayload), options);
-        } else if (settings.get().experimental.output === 'attribute') {
-            Object.keys(messagePayload).forEach((key) => {
-                if (typeof messagePayload[key] == 'object') {
-                    Object.keys(messagePayload[key]).forEach((subKey) => {
-                        this.mqtt.publish(`${entity.friendlyName}/${key}-${subKey}`,
-                            JSON.stringify(messagePayload[key][subKey]), options);
-                    });
-                } else {
-                    this.mqtt.publish(`${entity.friendlyName}/${key}`, JSON.stringify(messagePayload[key]), options);
+        if (isDevice && settings.get().mqtt.include_device_information) {
+            const attributes = [
+                'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
+                'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
+            ];
+
+            messagePayload.device = {
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
+            };
+
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        // filter mqtt message attributes
+        if (deviceOptions.filtered_attributes) {
+            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+
+        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity});
+
+        if (Object.entries(messagePayload).length) {
+            if (settings.get().experimental.output === 'attribute_and_json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+            } else if (settings.get().experimental.output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
+            } else {
+                /* istanbul ignore else */
+                if (settings.get().experimental.output === 'attribute') {
+                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
                 }
-            });
+            }
         }
     }
 
-    getDeviceInfoForMqtt(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        const {
-            type, nwkAddr, manufId, manufName, powerSource, modelId, hwVersion, swBuildId, status, dateCode,
-        } = device;
-        const deviceSettings = settings.getDevice(device.ieeeAddr);
+    async iteratePayloadAttributeOutput(topicRoot, payload, options) {
+        for (const [key, value] of Object.entries(payload)) {
+            let subPayload = value;
+            let message;
 
-        return {
-            ieeeAddr,
-            friendlyName: deviceSettings ? (deviceSettings.friendly_name || '') : '',
-            type,
-            nwkAddr,
-            manufId,
-            manufName,
-            powerSource,
-            modelId,
-            hwVersion: hwVersion ? hwVersion : 'unknown',
-            swBuildId: swBuildId ? swBuildId : 'unknown',
-            dateCode: dateCode ? dateCode : 'unknown',
-            status,
-        };
+            // Special cases
+            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
+                subPayload = [subPayload.r, subPayload.g, subPayload.b];
+            }
+
+            // Check Array first, since it is also an Object
+            if (Array.isArray(subPayload)) {
+                message = subPayload.map((x) => `${x}`).join(',');
+            } else if (typeof subPayload === 'object') {
+                return this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
+            } else {
+                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+            }
+
+            await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+        }
+    }
+
+    async callExtensionMethod(method, parameters) {
+        for (const extension of this.extensions) {
+            if (extension[method]) {
+                try {
+                    await extension[method](...parameters);
+                } catch (error) {
+                    /* istanbul ignore next */
+                    logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
+                    /* istanbul ignore next */
+                    if (process.env.JEST_WORKER_ID !== undefined) {
+                        throw error;
+                    }
+                }
+            }
+        }
     }
 }
 

lib/eventBus.js

@@ -0,0 +1,24 @@
+const events = require('events');
+const assert = require('assert');
+
+const allowedEvents = [
+    'deviceRemoved', // Device has been removed
+    'deviceRenamed', // Device has been renamed
+    'groupRenamed', // Group has been renamed
+    'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
+];
+
+class EventBus extends events.EventEmitter {
+    emit(event, data) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.emit(event, data);
+    }
+
+    on(event, callback) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.on(event, callback);
+    }
+}
+
+module.exports = EventBus;

lib/extension/baseExtension.js

@@ -1,8 +1,4 @@
-/**
- * This extensions is for documentation purposes only.
- * It describes all methods that are called by the controller.
- */
-class ExtensionTemplate {
+class BaseExtension {
     /**
      * Besides intializing variables, the constructor should do nothing!
      *
@@ -10,31 +6,34 @@ class ExtensionTemplate {
      * @param {MQTT} mqtt MQTT controller
      * @param {State} state State controller
      * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
      */
-    constructor(zigbee, mqtt, state, publishEntityState) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.zigbee = zigbee;
         this.mqtt = mqtt;
         this.state = state;
         this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
     }
 
     /**
      * This method is called by the controller once Zigbee has been started.
      */
-    onZigbeeStarted() {}
+    // onZigbeeStarted() {}
 
     /**
      * This method is called by the controller once connected to the MQTT server.
      */
-    onMQTTConnected() {}
+    // onMQTTConnected() {}
 
     /**
      * Is called when a Zigbee message from a device is received.
-     * @param {Object?} message The received message (can be null)
-     * @param {Object?} device The device of the message (can be null)
-     * @param {Object?} mappedDevice The mapped device (can be null)
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} settingsDevice Device settings
      */
-    onZigbeeMessage(message, device, mappedDevice) {}
+    // onZigbeeEvent(type, data, resolvedEntity) {}
 
     /**
      * Is called when a MQTT message is received
@@ -42,14 +41,12 @@ class ExtensionTemplate {
      * @param {Object} message The received message
      * @return {boolean} if the message was handled
      */
-    onMQTTMessage(topic, message) {
-        return false;
-    }
+    // onMQTTMessage(topic, message) {}
 
     /**
      * Is called once the extension has to stop
      */
-    stop() {}
+    // stop() {}
 }
 
-module.exports = ExtensionTemplate;
+module.exports = BaseExtension;

lib/extension/bridgeConfig.js

@@ -1,280 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const utils = require('../util/utils');
-
-const configRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+))`);
-const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
-
-class BridgeConfig {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-
-        // Bind functions
-        this.permitJoin = this.permitJoin.bind(this);
-        this.lastSeen = this.lastSeen.bind(this);
-        this.elapsed = this.elapsed.bind(this);
-        this.reset = this.reset.bind(this);
-        this.logLevel = this.logLevel.bind(this);
-        this.devices = this.devices.bind(this);
-        this.groups = this.groups.bind(this);
-        this.rename = this.rename.bind(this);
-        this.remove = this.remove.bind(this);
-        this.ban = this.ban.bind(this);
-        this.deviceOptions = this.deviceOptions.bind(this);
-        this.addGroup = this.addGroup.bind(this);
-        this.removeGroup = this.removeGroup.bind(this);
-
-        // Set supported options
-        this.supportedOptions = {
-            'permit_join': this.permitJoin,
-            'last_seen': this.lastSeen,
-            'elapsed': this.elapsed,
-            'reset': this.reset,
-            'log_level': this.logLevel,
-            'devices': this.devices,
-            'groups': this.groups,
-            'devices/get': this.devices,
-            'rename': this.rename,
-            'remove': this.remove,
-            'ban': this.ban,
-            'device_options': this.deviceOptions,
-            'add_group': this.addGroup,
-            'remove_group': this.removeGroup,
-        };
-    }
-
-    deviceOptions(topic, message) {
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error('Failed to parse message as JSON');
-            return;
-        }
-
-        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
-            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
-            return;
-        }
-
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(json.friendly_name);
-        if (!ieeeAddr) {
-            logger.error(`Failed to find device '${json.friendly_name}'`);
-            return;
-        }
-
-        settings.changeDeviceOptions(ieeeAddr, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
-    }
-
-    permitJoin(topic, message) {
-        this.zigbee.permitJoin(message.toString().toLowerCase() === 'true', () => this.publish());
-    }
-
-    reset(topic, message) {
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.error('Soft reset failed');
-            } else {
-                logger.info('Soft resetted ZNP');
-            }
-        });
-    }
-
-    lastSeen(topic, message) {
-        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'last_seen'], message);
-        logger.info(`Set last_seen to ${message}`);
-    }
-
-    elapsed(topic, message) {
-        const allowed = ['true', 'false'];
-        message = message.toString();
-
-        if (!allowed.includes(message)) {
-            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
-            return;
-        }
-
-        settings.set(['advanced', 'elapsed'], message === 'true');
-        logger.info(`Set elapsed to ${message}`);
-    }
-
-    logLevel(topic, message) {
-        const level = message.toString().toLowerCase();
-        if (allowedLogLevels.includes(level)) {
-            logger.info(`Switching log level to '${level}'`);
-            logger.transports.console.level = level;
-            logger.transports.file.level = level;
-        } else {
-            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
-        }
-
-        this.publish();
-    }
-
-    devices(topic, message) {
-        const devices = this.zigbee.getDevices().map((device) => {
-            const payload = {
-                ieeeAddr: device.ieeeAddr,
-                type: device.type,
-            };
-
-            if (device.type !== 'Coordinator') {
-                const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = mappedDevice ? mappedDevice.model : device.modelId;
-                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                payload.nwkAddr = device.nwkAddr;
-                payload.manufId = device.manufId;
-                payload.manufName = device.manufName;
-                payload.powerSource = device.powerSource;
-                payload.modelId = device.modelId;
-                payload.hwVersion = device.hwVersion;
-                payload.swBuildId = device.swBuildId;
-                payload.dateCode = device.dateCode;
-            }
-
-            return payload;
-        });
-
-        if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
-        } else {
-            this.mqtt.log('devices', devices);
-        }
-    }
-
-    groups(topic, message) {
-        this.mqtt.log('groups', settings.getGroups());
-    }
-
-    rename(topic, message) {
-        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} ` +
-                            `got ${message.toString()}`;
-
-        let json = null;
-        try {
-            json = JSON.parse(message.toString());
-        } catch (e) {
-            logger.error(invalid);
-            return;
-        }
-
-        // Validate message
-        if (!json.new || !json.old) {
-            logger.error(invalid);
-            return;
-        }
-
-        if (settings.changeFriendlyName(json.old, json.new)) {
-            logger.info(`Successfully renamed - ${json.old} to ${json.new} `);
-            this.mqtt.log('device_renamed', {from: json.old, to: json.new});
-        } else {
-            logger.error(`Failed to renamed - ${json.old} to ${json.new}`);
-            return;
-        }
-    }
-
-    addGroup(topic, message) {
-        const name = message.toString();
-        const added = settings.addGroup(name);
-        added ? logger.info(`Added group '${name}'`) : logger.error(`Failed to add group '${name}'`);
-    }
-
-    removeGroup(topic, message) {
-        const name = message.toString();
-        const removed = settings.removeGroup(name);
-        removed ? logger.info(`Removed group '${name}'`) : logger.error(`Failed to remove group '${name}'`);
-    }
-
-    remove(topic, message) {
-        this.removeOrBan(false, message);
-    }
-
-    ban(topic, message) {
-        this.removeOrBan(true, message);
-    }
-
-    removeOrBan(ban, message) {
-        message = message.toString();
-        const IDByFriendlyName = settings.getIeeeAddrByFriendlyName(message);
-        const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-        const device = this.zigbee.getDevice(deviceID);
-
-        const cleanup = () => {
-            // Remove from configuration.yaml
-            settings.removeDevice(deviceID);
-
-            // Remove from state
-            if (this.state) {
-                this.state.remove(deviceID);
-            }
-
-            logger.info(`Successfully ${ban ? 'banned' : 'removed'} ${deviceID}`);
-            this.mqtt.log(ban ? 'device_banned' : 'device_removed', message);
-        };
-
-        // Remove from zigbee network.
-        if (device) {
-            this.zigbee.removeDevice(deviceID, ban, (error) => {
-                if (!error) {
-                    cleanup();
-                } else {
-                    logger.error(`Failed to ${ban ? 'ban' : 'remove'} ${deviceID}`);
-                }
-            });
-        } else {
-            cleanup();
-        }
-    }
-
-    onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.publish();
-    }
-
-    onMQTTMessage(topic, message) {
-        if (!topic.match(configRegex)) {
-            return false;
-        }
-
-        const option = topic.match(configRegex)[1];
-
-        if (!this.supportedOptions.hasOwnProperty(option)) {
-            return false;
-        }
-
-        this.supportedOptions[option](topic, message);
-
-        return true;
-    }
-
-    publish() {
-        utils.getZigbee2mqttVersion((info) => {
-            const topic = `bridge/config`;
-            const payload = {
-                version: info.version,
-                commit: info.commitHash,
-                coordinator_firmware: this.zigbee.getFirmwareVersion(),
-                log_level: logger.transports.console.level,
-                permit_join: this.zigbee.getPermitJoin(),
-            };
-
-            this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0}, null);
-        });
-    }
-}
-
-module.exports = BridgeConfig;

lib/extension/bridgeLegacy.js

@@ -0,0 +1,462 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const utils = require('../util/utils');
+const assert = require('assert');
+const BaseExtension = require('./baseExtension');
+
+const configRegex =
+    new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
+const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
+
+class BridgeLegacy extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        // Bind functions
+        this.permitJoin = this.permitJoin.bind(this);
+        this.lastSeen = this.lastSeen.bind(this);
+        this.elapsed = this.elapsed.bind(this);
+        this.reset = this.reset.bind(this);
+        this.logLevel = this.logLevel.bind(this);
+        this.devices = this.devices.bind(this);
+        this.groups = this.groups.bind(this);
+        this.rename = this.rename.bind(this);
+        this.renameLast = this.renameLast.bind(this);
+        this.remove = this.remove.bind(this);
+        this.forceRemove = this.forceRemove.bind(this);
+        this.ban = this.ban.bind(this);
+        this.deviceOptions = this.deviceOptions.bind(this);
+        this.addGroup = this.addGroup.bind(this);
+        this.removeGroup = this.removeGroup.bind(this);
+        this.whitelist = this.whitelist.bind(this);
+        this.touchlinkFactoryReset = this.touchlinkFactoryReset.bind(this);
+
+        this.lastJoinedDeviceName = null;
+
+        // Set supported options
+        this.supportedOptions = {
+            'permit_join': this.permitJoin,
+            'last_seen': this.lastSeen,
+            'elapsed': this.elapsed,
+            'reset': this.reset,
+            'log_level': this.logLevel,
+            'devices': this.devices,
+            'groups': this.groups,
+            'devices/get': this.devices,
+            'rename': this.rename,
+            'rename_last': this.renameLast,
+            'remove': this.remove,
+            'force_remove': this.forceRemove,
+            'ban': this.ban,
+            'device_options': this.deviceOptions,
+            'add_group': this.addGroup,
+            'remove_group': this.removeGroup,
+            'whitelist': this.whitelist,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+        };
+    }
+
+    whitelist(topic, message) {
+        try {
+            const entity = settings.getEntity(message);
+            assert(entity, `Entity '${message}' does not exist`);
+            settings.whitelistDevice(entity.ID);
+            logger.info(`Whitelisted '${entity.friendlyName}'`);
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to whitelist '${message}' '${error}'`);
+        }
+    }
+
+    deviceOptions(topic, message) {
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error('Failed to parse message as JSON');
+            return;
+        }
+
+        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
+            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
+            return;
+        }
+
+        const entity = settings.getEntity(json.friendly_name);
+        assert(entity, `Entity '${json.friendly_name}' does not exist`);
+        settings.changeDeviceOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
+    }
+
+    async permitJoin(topic, message) {
+        await this.zigbee.permitJoin(message.toLowerCase() === 'true');
+        this.publish();
+    }
+
+    async reset(topic, message) {
+        try {
+            await this.zigbee.reset('soft');
+            logger.info('Soft resetted ZNP');
+        } catch (error) {
+            logger.error('Soft reset failed');
+        }
+    }
+
+    lastSeen(topic, message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        logger.info(`Set last_seen to ${message}`);
+    }
+
+    elapsed(topic, message) {
+        const allowed = ['true', 'false'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'elapsed'], message === 'true');
+        logger.info(`Set elapsed to ${message}`);
+    }
+
+    logLevel(topic, message) {
+        const level = message.toLowerCase();
+        if (allowedLogLevels.includes(level)) {
+            logger.info(`Switching log level to '${level}'`);
+            logger.setLevel(level);
+        } else {
+            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
+        }
+
+        this.publish();
+    }
+
+    async devices(topic, message) {
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const devices = this.zigbee.getDevices().map((device) => {
+            const payload = {
+                ieeeAddr: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+            };
+
+            if (device.type !== 'Coordinator') {
+                const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const friendlyDevice = settings.getDevice(device.ieeeAddr);
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
+                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+                payload.manufacturerID = device.manufacturerID;
+                payload.manufacturerName = device.manufacturerName;
+                payload.powerSource = device.powerSource;
+                payload.modelID = device.modelID;
+                payload.hardwareVersion = device.hardwareVersion;
+                payload.softwareBuildID = device.softwareBuildID;
+                payload.dateCode = device.dateCode;
+                payload.lastSeen = device.lastSeen;
+            } else {
+                payload.friendly_name = 'Coordinator';
+                payload.softwareBuildID = coordinator.type;
+                payload.dateCode = coordinator.meta.revision.toString();
+                payload.lastSeen = Date.now();
+            }
+
+            return payload;
+        });
+
+        if (topic.split('/').pop() == 'get') {
+            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
+        } else {
+            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
+        }
+    }
+
+    groups(topic, message) {
+        const payload = settings.getGroups().map((g) => {
+            const group = {...g};
+            delete group.friendlyName;
+            return group;
+        });
+
+        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
+    }
+
+    rename(topic, message) {
+        const invalid = `Invalid rename message format expected {old: 'friendly_name', new: 'new_name} got ${message}`;
+
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error(invalid);
+            return;
+        }
+
+        // Validate message
+        if (!json.new || !json.old) {
+            logger.error(invalid);
+            return;
+        }
+
+        this._renameInternal(json.old, json.new);
+    }
+
+    renameLast(topic, message) {
+        if (!this.lastJoinedDeviceName) {
+            logger.error(`Cannot rename last joined device, no device has joined during this session`);
+            return;
+        }
+
+        this._renameInternal(this.lastJoinedDeviceName, message);
+    }
+
+    _renameInternal(from, to) {
+        try {
+            const isGroup = settings.getGroup(from) !== null;
+            settings.changeFriendlyName(from, to);
+            logger.info(`Successfully renamed - ${from} to ${to} `);
+            const entity = this.zigbee.resolveEntity(to);
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to rename - ${from} to ${to}`);
+        }
+    }
+
+    addGroup(topic, message) {
+        let id = null;
+        let name = null;
+        try {
+            // json payload with id and friendly_name
+            const json = JSON.parse(message);
+            if (json.hasOwnProperty('id')) {
+                id = json.id;
+                name = `group_${id}`;
+            }
+            if (json.hasOwnProperty('friendly_name')) {
+                name = json.friendly_name;
+            }
+        } catch (e) {
+            // just friendly_name
+            name = message;
+        }
+
+        if (name == null) {
+            logger.error('Failed to add group, missing friendly_name!');
+            return;
+        }
+
+        const group = settings.addGroup(name, id);
+        this.zigbee.createGroup(group.ID);
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
+        logger.info(`Added group '${name}'`);
+    }
+
+    removeGroup(topic, message) {
+        const name = message;
+        const entity = this.zigbee.resolveEntity(message);
+        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+        settings.removeGroup(message);
+        entity.group.removeFromDatabase();
+        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
+        logger.info(`Removed group '${name}'`);
+    }
+
+    async forceRemove(topic, message) {
+        await this.removeForceRemoveOrBan('force_remove', message);
+    }
+
+    async remove(topic, message) {
+        await this.removeForceRemoveOrBan('remove', message);
+    }
+
+    async ban(topic, message) {
+        await this.removeForceRemoveOrBan('ban', message);
+    }
+
+    async removeForceRemoveOrBan(action, message) {
+        const entity = this.zigbee.resolveEntity(message.trim());
+        const lookup = {
+            ban: ['banned', 'Banning', 'ban'],
+            force_remove: ['force_removed', 'Force removing', 'force remove'],
+            remove: ['removed', 'Removing', 'remove'],
+        };
+
+        if (!entity) {
+            logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            return;
+        }
+
+        const cleanup = () => {
+            // Fire event
+            this.eventBus.emit('deviceRemoved', {device: entity.device});
+
+            // Remove from configuration.yaml
+            settings.removeDevice(entity.settings.ID);
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
+        };
+
+        try {
+            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            if (action === 'force_remove') {
+                await entity.device.removeFromDatabase();
+            } else {
+                await entity.device.removeFromNetwork();
+            }
+
+            cleanup();
+        } catch (error) {
+            logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+            // eslint-disable-next-line
+            logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+        }
+
+        if (action === 'ban') {
+            settings.banDevice(entity.settings.ID);
+        }
+    }
+
+    async onMQTTConnected() {
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
+        await this.publish();
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (!topic.match(configRegex)) {
+            return false;
+        }
+
+        const option = topic.match(configRegex)[1];
+
+        if (!this.supportedOptions.hasOwnProperty(option)) {
+            return false;
+        }
+
+        await this.supportedOptions[option](topic, message);
+
+        return true;
+    }
+
+    async publish() {
+        const info = await utils.getZigbee2mqttVersion();
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const topic = `bridge/config`;
+        const payload = {
+            version: info.version,
+            commit: info.commitHash,
+            coordinator,
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
+        if (type === 'deviceJoined') {
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        logger.info('Starting touchlink factory reset...');
+        this.mqtt.publish(
+            'bridge/log',
+            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
+        const result = await this.zigbee.touchlinkFactoryReset();
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
+        } else {
+            logger.warn('Failed to factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
+        }
+    }
+}
+
+module.exports = BridgeLegacy;

lib/extension/deviceAvailability.js

@@ -1,156 +1,213 @@
 const logger = require('../util/logger');
 const settings = require('../util/settings');
 const utils = require('../util/utils');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const BaseExtension = require('./baseExtension');
 
 // Some EndDevices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableDevices = [
-    zigbeeShepherdConverters.devices.find((d) => d.model === 'E11-G13'),
+const pingableModels = [
+    'E11-G13',
+    '53170161',
 ];
 
-const toZigbeeCandidates = ['state'];
+const forcedPingable =
+    zigbeeHerdsmanConverters.devices.filter((d) => pingableModels.includes(d.model));
+
+const toZigbeeCandidates = ['state', 'brightness', 'color', 'color_temp'];
+
+const Hours25 = 1000 * 60 * 60 * 25;
 
 /**
  * This extensions pings devices to check if they are online.
  */
-class DeviceAvailability {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
+class DeviceAvailability extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
         this.availability_timeout = settings.get().advanced.availability_timeout;
         this.timers = {};
-        this.pending = [];
         this.state = {};
 
         // Initialize blacklist
         this.blacklist = settings.get().advanced.availability_blacklist.map((e) => {
-            return settings.getIeeeAddrByFriendlyName(e) || e;
+            return settings.getEntity(e).ID;
+        });
+
+        // Initialize whitelist
+        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => {
+            return settings.getEntity(e).ID;
         });
     }
 
-    isPingable(device) {
-        if (this.blacklist.includes(device.ieeeAddr)) {
+    isAllowed(device) {
+        const ieeeAddr = device.ieeeAddr;
+
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings.friendly_name;
+
+        // Whitelist is not empty and device is in it, enable availability
+        if (this.whitelist.length > 0) {
+            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        }
+
+        // Device is on blacklist, disable availability
+        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
             return false;
         }
 
-        if (pingableDevices.find((d) => d.zigbeeModel.includes(device.modelId))) {
+        return true;
+    }
+
+    isPingable(device) {
+        // Device is on forcedPingable-list, enable availability
+        if (forcedPingable.find((d) => d.zigbeeModel.includes(device.modelID))) {
             return true;
         }
 
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
-    }
+        // Device is a mains powered router
+        const result = utils.isRouter(device) && !utils.isBatteryPowered(device);
 
-    getAllPingableDevices() {
-        return this.zigbee.getAllClients().filter((d) => this.isPingable(d));
+        return result;
     }
 
     onMQTTConnected() {
-        // As some devices are not checked for availability (e.g. battery powered devices)
-        // we mark all device as online by default.
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, true));
+        for (const device of this.zigbee.getClients()) {
+            // Mark all devices as online on start
+            this.publishAvailability(device, true);
 
-        // Start timers for all devices
-        this.getAllPingableDevices().forEach((device) => this.setTimer(device));
+            if (this.isAllowed(device)) {
+                if (this.isPingable(device)) {
+                    this.setTimerPingable(device);
+                } else {
+                    this.timers[device.ieeeAddr] = setInterval(() => {
+                        this.handleIntervalNotPingable(device);
+                    }, utils.secondsToMilliseconds(300));
+                }
+            }
+        }
     }
 
-    handleInterval(device) {
-        // Check if a job is already pending.
-        // This avoids overflowing of the queue in case the queue is not able to catch-up with the jobs being added.
-        const ieeeAddr = device.ieeeAddr;
-        if (this.pending.includes(ieeeAddr)) {
-            logger.debug(`Skipping ping for ${ieeeAddr} becuase job is already in queue`);
+    async handleIntervalPingable(device) {
+        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
+        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!entity) {
+            logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
             return;
         }
 
-        this.pending.push(ieeeAddr);
-
-        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const errorLogLevel = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
-        const mechanism = utils.isXiaomiDevice(device) ? 'basic' : 'default';
-
-        this.zigbee.ping(ieeeAddr, errorLogLevel, (error) => {
-            this.publishAvailability(ieeeAddr, !error);
-
-            // Remove from pending jobs.
-            const index = this.pending.indexOf(ieeeAddr);
-            if (index !== -1) {
-                this.pending.splice(index, 1);
-            }
-
-            this.setTimer(device);
-        }, mechanism);
+        const ieeeAddr = device.ieeeAddr;
+        const level = this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] ? 'debug' : 'error';
+        try {
+            await device.ping();
+            this.publishAvailability(device, true);
+            logger.debug(`Successfully pinged '${entity.name}'`);
+        } catch (error) {
+            this.publishAvailability(device, false);
+            logger[level](`Failed to ping '${entity.name}'`);
+        } finally {
+            this.setTimerPingable(device);
+        }
     }
 
-    setTimer(device) {
+    async handleIntervalNotPingable(device) {
+        const ago = Date.now() - device.lastSeen;
+        const entity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!entity || !device.lastSeen) {
+            return;
+        }
+
+        logger.debug(`Non-pingable device '${entity.name}' was last seen '${ago / 1000}' seconds ago.`);
+
+        if (ago > Hours25) {
+            this.publishAvailability(device, false);
+        }
+    }
+
+    setTimerPingable(device) {
         if (this.timers[device.ieeeAddr]) {
             clearTimeout(this.timers[device.ieeeAddr]);
         }
 
-        this.timers[device.ieeeAddr] = setTimeout(() => {
-            this.handleInterval(device);
+        this.timers[device.ieeeAddr] = setTimeout(async () => {
+            await this.handleIntervalPingable(device);
         }, utils.secondsToMilliseconds(this.availability_timeout));
     }
 
-    stop() {
-        this.zigbee.getDevices()
-            .filter((d) => d.type !== 'Coordinator')
-            .forEach((device) => this.publishAvailability(device.ieeeAddr, false));
-    }
-
-    onReconnect(ieeeAddr) {
-        const device = this.zigbee.getDevice(ieeeAddr);
-        let mappedDevice = null;
-
-        if (device && device.modelId) {
-            mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+    async stop() {
+        for (const timer of Object.values(this.timers)) {
+            clearTimeout(timer);
         }
 
-        if (mappedDevice) {
-            const converters = mappedDevice.toZigbee.filter((tz) => {
-                return tz.key.find((k) => toZigbeeCandidates.includes(k));
-            });
+        this.zigbee.getClients().forEach((device) => this.publishAvailability(device, false));
+    }
 
-            converters.forEach((converter) => {
-                const converted = converter.convert(null, null, null, 'get');
-                if (converted) {
-                    this.zigbee.publish(
-                        ieeeAddr, 'device', converted.cid, converted.cmd, converted.cmdType,
-                        converted.zclData, converted.cfg, null, () => {}
-                    );
+    async onReconnect(device) {
+        if (device && device.modelID) {
+            const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+
+            if (definition) {
+                const used = [];
+                try {
+                    for (const key of toZigbeeCandidates) {
+                        const converter = definition.toZigbee.find((tz) => tz.key.includes(key));
+                        if (converter && !used.includes(converter)) {
+                            await converter.convertGet(device.endpoints[0], key, {});
+                            used.push(converter);
+                        }
+                    }
+                } catch (error) {
+                    const entity = this.zigbee.resolveEntity(device.ieeeAddr);
+                    logger.error(`Failed to read state of '${entity.name}' after reconnect`);
                 }
-            });
+            }
         }
     }
 
-    publishAvailability(ieeeAddr, available) {
+    publishAvailability(device, available) {
+        const ieeeAddr = device.ieeeAddr;
         if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
-            this.onReconnect(ieeeAddr);
+            this.onReconnect(device);
         }
 
-        this.state[ieeeAddr] = available;
         const deviceSettings = settings.getDevice(ieeeAddr);
         const name = deviceSettings ? deviceSettings.friendly_name : ieeeAddr;
         const topic = `${name}/availability`;
         const payload = available ? 'online' : 'offline';
-        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+        if (this.state[ieeeAddr] !== available) {
+            this.state[ieeeAddr] = available;
+            this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+        }
     }
 
-    onZigbeeMessage(message, device, mappedDevice) {
-        // When a zigbee message from a device is received we know the device is still alive.
-        // => reset the timer.
-        if (device && this.isPingable(this.zigbee.getDevice(device.ieeeAddr))) {
-            // When a message is received and the device is marked as offline, mark it online.
-            if (this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr]) {
-                this.publishAvailability(device.ieeeAddr, true);
-            } else if (!this.state.hasOwnProperty(device.ieeeAddr)) {
-                // A new device has been connected
-                this.publishAvailability(device.ieeeAddr, true);
-            }
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+        if (!device) {
+            return;
+        }
 
-            this.setTimer(device);
+        if (this.isAllowed(device)) {
+            this.publishAvailability(data.device, true);
+
+            if (this.isPingable(device)) {
+                // When a zigbee message from a device is received we know the device is still alive.
+                // => reset the timer.
+                this.setTimerPingable(device);
+
+                const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
+                if (online && type === 'deviceAnnounce' && !utils.isIkeaTradfriDevice(device)) {
+                    /**
+                     * In case the device is powered off AND on within the availability timeout,
+                     * zigbee2qmtt does not detect the device as offline (device is still marked online).
+                     * When a device is turned on again the state could be out of sync.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
+                     * deviceAnnounce is typically send when a device comes online.
+                     *
+                     * This isn't needed for TRADFRI devices as they already send the state themself.
+                     */
+                    this.onReconnect(device);
+                }
+            }
         }
     }
 }

lib/extension/deviceBind.js

@@ -1,124 +1,105 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-const postfixes = utils.getPostfixes();
+const assert = require('assert');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const BaseExtension = require('./baseExtension');
 
-const allowedClusters = [
-    5, // genScenes
-    6, // genOnOff
-    8, // genLevelCtrl
-    768, // lightingColorCtrl
-];
+const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
 
-class DeviceBind {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
+// See zigbee-herdsman-converters devices.js
+const defaultBindGroup = {type: 'group_number', ID: 901};
 
+class DeviceBind extends BaseExtension {
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/+/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/+/+`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
     }
 
-    getIDAndPostfix(topic) {
-        let postfix = null;
-        if (postfixes.find((p) => topic.endsWith(`/${p}`))) {
-            postfix = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-            // Remove postfix from topic
-            topic = topic.replace(`/${postfix}`, '');
-        }
-
-        return {ID: topic, postfix};
-    }
-
-    parseTopic(topic) {
+    async onMQTTMessage(topic, message) {
         if (!topic.match(topicRegex)) {
             return null;
         }
 
-        // Remove base from topic
+        // Parse topic, retrieve type (bind or unbind) and source
         topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-
-        // Parse type from topic
         const type = topic.split('/')[0];
+        const sourceKey = topic.replace(`${type}/`, '');
+        const targetKey = message;
 
-        // Remove type from topic
-        topic = topic.replace(`${type}/`, '');
-        return {type, ...this.getIDAndPostfix(topic)};
-    }
+        // Find source; can only be a device and target
+        const source = this.zigbee.resolveEntity(sourceKey);
+        assert(source != null && source.type === 'device', 'Source undefined or not a device');
+        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        assert(target != null, 'Target is unknown');
 
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Find source; can only be a device.
-        const sourceEntity = settings.resolveEntity(topic.ID);
-        const source = utils.getEndpointByEntityID(this.zigbee, sourceEntity.ID, topic.postfix);
-
-        const targetEntityIDPostfix= this.getIDAndPostfix(message.toString());
-        const targetEntity = settings.resolveEntity(targetEntityIDPostfix.ID);
-        let target = null;
-        if (targetEntity.type === 'device') {
-            target = utils.getEndpointByEntityID(this.zigbee, targetEntity.ID, targetEntityIDPostfix.postfix);
-        } else if (targetEntity.type === 'group') {
-            target = targetEntity.ID;
-        }
-
-        if (!source || !target) {
-            return false;
-        }
+        const sourceName = source.settings.friendlyName;
+        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+        let attemptedToBindSomething = false;
 
         // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters (as we don't know which devices are in)
-        let supported = [];
-        if (targetEntity.type === 'device') {
-            supported = target.getSimpleDesc().inClusterList.filter((cluster) => {
-                return allowedClusters.includes(cluster);
-            });
-        } else if (targetEntity.type === 'group') {
-            supported = allowedClusters;
+        // Groups are assumed to support all clusters.
+        for (const cluster of clusters) {
+            const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+
+            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                attemptedToBindSomething = true;
+                try {
+                    let bindTarget = null;
+                    if (target.type === 'group') bindTarget = target.group;
+                    else if (target.type === 'group_number') bindTarget = target.ID;
+                    else bindTarget = target.endpoint;
+
+                    if (type === 'bind') {
+                        await source.endpoint.bind(cluster, bindTarget);
+                    } else {
+                        await source.endpoint.unbind(cluster, bindTarget);
+                    }
+
+                    logger.info(
+                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                        `'${sourceName}' to '${targetName}'`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const message = {from: sourceName, to: targetName, cluster};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}`, message}),
+                        );
+                    }
+                } catch (error) {
+                    logger.error(
+                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                        `'${targetName}' (${error})`,
+                    );
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const message = {from: sourceName, to: targetName, cluster};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            JSON.stringify({type: `device_${type}_failed`, message}),
+                        );
+                    }
+                }
+            }
         }
 
-        const clusters = source.getSimpleDesc().outClusterList.filter((cluster) => {
-            return supported.includes(cluster);
-        });
+        if (!attemptedToBindSomething) {
+            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
 
-        // Bind
-        clusters.forEach((cluster) => {
-            logger.debug(`${topic.type}ing cluster '${cluster}' from ${sourceEntity.ID}' to '${targetEntity.ID}'`);
-
-            this.zigbee[topic.type](source, cluster, target, (error) => {
-                if (error) {
-                    logger.error(
-                        `Failed to ${topic.type} cluster '${cluster}' from ${sourceEntity.ID}' to ` +
-                        `'${targetEntity.ID}' (${error})`
-                    );
-                } else {
-                    logger.info(
-                        `Successfully ${topic.type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `${sourceEntity.ID}' to '${targetEntity.ID}'`
-                    );
-
-                    this.mqtt.log(
-                        `device_${topic.type}`,
-                        {from: sourceEntity.ID, to: targetEntity.ID, cluster}
-                    );
-                }
-            });
-        });
-
-        return true;
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {from: sourceName, to: targetName};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_${type}_failed`, message}),
+                );
+            }
+        }
     }
 }
 

lib/extension/deviceConfigure.js

@@ -1,89 +1,109 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const Queue = require('queue');
+const BaseExtension = require('./baseExtension');
 
-/**
- * This extensions handles configuration of devices.
- */
-class DeviceConfigure {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.configured = {};
+class DeviceConfigure extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
         this.attempts = {};
-
-        this.queue = new Queue();
-        this.queue.concurrency = 1;
-        this.queue.autostart = true;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/configure`;
     }
 
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => {
-            const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
 
-            if (mappedDevice) {
-                this.configure(device.ieeeAddr, mappedDevice);
-            }
-        });
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
     }
 
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (device && mappedDevice) {
-            const ieeeAddr = device.ieeeAddr;
-            if (!this.configured[ieeeAddr]) {
-                this.configure(ieeeAddr, mappedDevice);
+    onMQTTConnected() {
+        this.mqtt.subscribe(this.topic);
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (topic !== this.topic) {
+            return;
+        }
+
+        const resolvedEntity = this.zigbee.resolveEntity(message);
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            logger.error(`Device '${message}' does not exist`);
+            return;
+        }
+
+        if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+            return;
+        }
+
+        this.configure(resolvedEntity, true);
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
             }
         }
     }
 
-    configure(ieeeAddr, mappedDevice) {
-        if (!this.attempts.hasOwnProperty(ieeeAddr)) {
-            this.attempts[ieeeAddr] = 0;
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
         }
 
-        if (mappedDevice.configure) {
-            const friendlyName = settings.getDevice(ieeeAddr) ? settings.getDevice(ieeeAddr).friendly_name : 'unknown';
-            // mark device as currently being configured to prevent race conditions
-            this.configured[ieeeAddr] = 'configuring';
-
-            this.queue.push((queueCallback) => {
-                logger.debug(`Configuring ${friendlyName} (${ieeeAddr}) ...`);
-
-                // Call configure function of this device.
-                mappedDevice.configure(ieeeAddr, this.zigbee.shepherd, this.zigbee.getCoordinator(), (ok, msg) => {
-                    if (ok) {
-                        logger.info(`Successfully configured ${friendlyName} (${ieeeAddr})`);
-                        this.configured[ieeeAddr] = 'configured';
-                    } else {
-                        // If a device is not joined in the network the configure command may fail
-                        // (especially, immediately after zigbee start)
-                        // also, this command may fail if the network is busy or lost data packets
-                        if (this.attempts[ieeeAddr] > 1) {
-                            // Give up after 3 attempts.
-                            this.configured[ieeeAddr] = 'failed';
-                        } else {
-                            // we're not done yet so mark not busy
-                            delete this.configured[ieeeAddr];
-                        }
-
-                        logger.warn(
-                            `Failed to configure ${friendlyName} (${ieeeAddr}) ('${msg}')` +
-                            ` (attempt #${this.attempts[ieeeAddr] + 1})`
-                        );
-
-                        logger.warn(`This can be ignored if the device is working properly`);
-
-                        this.attempts[ieeeAddr] += 1;
-                    }
-
-                    queueCallback();
-                });
-            });
-        } else {
-            // no need to configure anyting. So just mark as done
-            this.configured[ieeeAddr] = 'configured';
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
         }
     }
+
+    async configure(resolvedEntity, force=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+        } catch (error) {
+            logger.error(
+                `Failed to configure '${resolvedEntity.name}', ` +
+                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
+            );
+            this.attempts[device.ieeeAddr]++;
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
 }
 
 module.exports = DeviceConfigure;

lib/extension/deviceEvent.js

@@ -0,0 +1,36 @@
+const BaseExtension = require('./baseExtension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+class DeviceEvent extends BaseExtension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            this.callOnEvent(device, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.device && resolvedEntity.definition) {
+            this.callOnEvent(data.device, type, data, resolvedEntity.definition);
+        }
+    }
+
+    async stop() {
+        for (const device of this.zigbee.getClients()) {
+            this.callOnEvent(device, 'stop', {});
+        }
+    }
+
+    callOnEvent(device, type, data, definition) {
+        zigbeeHerdsmanConverters.onEvent(type, data, device);
+
+        if (!definition) {
+            definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+        }
+
+        if (definition && definition.onEvent) {
+            definition.onEvent(type, data, device);
+        }
+    }
+}
+
+module.exports = DeviceEvent;

lib/extension/deviceGroupMembership.js

@@ -1,72 +1,45 @@
+/* istanbul ignore file */
 const settings = require('../util/settings');
 const logger = require('../util/logger');
+const BaseExtension = require('./baseExtension');
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/.+/get_group_membership$`);
-
-class DeviceGroupMembership {
-    constructor(zigbee, mqtt, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.publishEntityState = publishEntityState;
-    }
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
+class DeviceGroupMembership extends BaseExtension {
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/device/+/get_group_membership`);
+        for (let step = 1; step < 20; step++) {
+            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
+            this.mqtt.subscribe(topic);
+        }
     }
 
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(topicRegex);
+        if (!match) {
             return null;
         }
 
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/device/`, '');
-
-        // Remove command from topic
-        topic = topic.replace(`/get_group_membership`, '');
-
-        return {friendlyName: topic};
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Map message to ieeeAddr and check if device exist.
-        const friendlyName = topic.friendlyName;
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(friendlyName) || friendlyName;
-        if (!this.zigbee.getDevice(ieeeAddr)) {
-            logger.error(`Failed to find device '${friendlyName}'`);
+        const entity = this.zigbee.resolveEntity(match[1]);
+        if (!entity || entity.type !== 'device') {
+            logger.error(`Device '${match[1]}' does not exist`);
             return;
         }
 
-        const callback = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to get membership of ${ieeeAddr}`);
-            } else {
-                const {grouplist, capacity} = rsp;
-                const msgGroupList = `${ieeeAddr} is in groups [${grouplist}]`;
-                let msgCapacity;
-                if (capacity === 254) {
-                    msgCapacity = 'it can be a part of at least 1 more group';
-                } else {
-                    msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
-                }
-                logger.info(`${msgGroupList} and ${msgCapacity}`);
-
-                this.publishEntityState(ieeeAddr, {group_list: grouplist, group_capacity: capacity});
-            }
-        };
-
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', 'getMembership', 'functional',
-            {groupcount: 0, grouplist: []}, null, null, callback,
+        const response = await entity.endpoint.command(
+            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
         );
 
-        return true;
+        const {grouplist, capacity} = response;
+        const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
+        let msgCapacity;
+        if (capacity === 254) {
+            msgCapacity = 'it can be a part of at least 1 more group';
+        } else {
+            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
+        }
+        logger.info(`${msgGroupList} and ${msgCapacity}`);
+
+        this.publishEntityState(entity.device.ieeeAddr, {group_list: grouplist, group_capacity: capacity});
     }
 }
 

lib/extension/devicePublish.js

@@ -1,228 +0,0 @@
-
-const settings = require('../util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/.+/(set|get)$`);
-const postfixes = utils.getPostfixes();
-
-const maxDepth = 20;
-
-const groupConverters = [
-    zigbeeShepherdConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeShepherdConverters.toZigbeeConverters.light_colortemp,
-    zigbeeShepherdConverters.toZigbeeConverters.light_color,
-    zigbeeShepherdConverters.toZigbeeConverters.ignore_transition,
-];
-
-class DevicePublish {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-    }
-
-    onMQTTConnected() {
-        // Subscribe to topics.
-        for (let step = 1; step < maxDepth; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}get`);
-        }
-    }
-
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/`, '');
-        if (topic.match(/bridge\/config/)) {
-            return null;
-        }
-        // Parse type from topic
-        const type = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-        // Remove type from topic
-        topic = topic.replace(`/${type}`, '');
-
-        // Check if we have to deal with a postfix.
-        let postfix = '';
-        if (postfixes.find((p) => topic.endsWith(`/${p}`))) {
-            postfix = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-            // Remove postfix from topic
-            topic = topic.replace(`/${postfix}`, '');
-        }
-
-        const ID = topic;
-
-        return {type, ID, postfix};
-    }
-
-    handlePublishError(entity, message, error) {
-        const meta = {
-            entity,
-            message: message.toString(),
-        };
-
-        this.mqtt.log('zigbee_publish_error', error.toString(), meta);
-    }
-
-    handlePublished(entity, topic, converter, converted, key, value) {
-        if (topic.type === 'set' && converted.hasOwnProperty('newState') && converted.newState) {
-            const msg = converted.newState;
-
-            if (topic.postfix) {
-                msg[`state_${topic.postfix}`] = msg.state;
-                delete msg.state;
-            }
-
-            this.publishEntityState(entity.ID, msg);
-        }
-    }
-
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-
-        if (!topic) {
-            return false;
-        }
-
-        // Resolve the entity
-        const entity = settings.resolveEntity(topic.ID);
-
-        // Get entity details
-        let endpoint = null;
-        let converters = null;
-        let device = null;
-
-        if (entity.type === 'device') {
-            device = this.zigbee.getDevice(entity.ID);
-            if (!device) {
-                logger.error(`Failed to find device with ieeAddr: '${entity.ID}'`);
-                this.mqtt.log('entity_not_found', entity.ID);
-                return;
-            }
-
-            // Map device to a model
-            const model = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (!model) {
-                logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-                return;
-            }
-
-            // Determine endpoint to publish to.
-            if (model.hasOwnProperty('ep')) {
-                const eps = model.ep(device);
-                endpoint = eps.hasOwnProperty(topic.postfix) ? eps[topic.postfix] : null;
-                if (endpoint === null && eps.hasOwnProperty('default')) {
-                    endpoint = eps['default'];
-                }
-            }
-
-            converters = model.toZigbee;
-        } else if (entity.type === 'group') {
-            converters = groupConverters;
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = null;
-        try {
-            json = JSON.parse(message);
-        } catch (e) {
-            // Cannot be parsed to JSON, assume state message.
-            json = {state: message.toString()};
-        }
-
-        /**
-         * Home Assistant always publishes 'state', even when e.g. only setting
-         * the color temperature. This would lead to 2 zigbee publishes, where the first one
-         * (state) is probably unecessary.
-         */
-        if (settings.get().homeassistant && this.state) {
-            const deviceState = this.state.get(entity.ID);
-            const hasColorTemp = json.hasOwnProperty('color_temp');
-            const hasColor = json.hasOwnProperty('color');
-            const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
-            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
-                delete json.state;
-                logger.debug('Skipping state because of Home Assistant');
-            }
-        }
-
-        // Ensure that state and brightness are executed before other commands.
-        const keys = Object.keys(json);
-        keys.sort((a, b) => (['state', 'brightness'].includes(a) ? -1 : 1));
-
-        // For each key in the JSON message find the matching converter.
-        const usedConverters = [];
-        keys.forEach((key) => {
-            const converter = converters.find((c) => c.key.includes(key));
-
-            if (usedConverters.includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
-                return;
-            }
-
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                return;
-            }
-
-            // Converter didn't return a result, skip
-            const converted = converter.convert(key, json[key], json, topic.type, topic.postfix);
-            if (!converted) {
-                return;
-            }
-
-            this.zigbee.publish(
-                entity.ID,
-                entity.type,
-                converted.cid,
-                converted.cmd,
-                converted.cmdType,
-                converted.zclData,
-                converted.cfg,
-                endpoint,
-                (error, rsp) => {
-                    if (!error) {
-                        this.handlePublished(entity, topic, converter, converted, key, json[key]);
-                    } else {
-                        this.handlePublishError(entity, message, error);
-                    }
-                }
-            );
-
-            // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-            // So here we re-read the value after a specified timeout, this timeout could for example be the
-            // transition time of a color change or for forcing a state read for devices that don't
-            // automatically report a new state when set.
-            // When reporting is requested for a device (report: true in device-specific settings) we won't
-            // ever issue a read here, as we assume the device will properly report changes.
-            // Only do this when the retrieve_state option is enabled for this device.
-            const deviceSettings = settings.getDevice(entity.ID);
-            if (topic.type === 'set' && entity.type === 'device' && converted.hasOwnProperty('readAfterWriteTime') &&
-                deviceSettings && deviceSettings.retrieve_state) {
-                const getConverted = converter.convert(key, json[key], json, 'get');
-                setTimeout(() => {
-                    this.zigbee.publish(
-                        entity.ID, entity.type, getConverted.cid, getConverted.cmd, getConverted.cmdType,
-                        getConverted.zclData, getConverted.cfg, endpoint, () => {}
-                    );
-                }, converted.readAfterWriteTime);
-            }
-
-            usedConverters.push(converter);
-        });
-
-        return true;
-    }
-}
-
-module.exports = DevicePublish;

lib/extension/deviceReceive.js

@@ -2,26 +2,21 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
+const BaseExtension = require('./baseExtension');
 
-/**
- * This extensions handles messages received from devices.
- */
-class DeviceReceive {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
+class DeviceReceive extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.coordinator = null;
         this.elapsed = {};
         this.debouncers = {};
     }
 
-    onZigbeeStarted() {
-        this.coordinator = this.zigbee.getCoordinator().device.ieeeAddr;
+    async onZigbeeStarted() {
+        this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
     }
 
-    publishDebounce(ieeeAddr, payload, time) {
+    publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
         if (!this.debouncers[ieeeAddr]) {
             this.debouncers[ieeeAddr] = {
                 payload: {},
@@ -32,52 +27,41 @@ class DeviceReceive {
             };
         }
 
+        if (this.isPayloadConflicted(payload, this.debouncers[ieeeAddr].payload, debounceIgnore)) {
+            // publish previous payload immediately
+            this.debouncers[ieeeAddr].publish.flush();
+        }
+
+        // extend debounced payload with current
         this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
         this.debouncers[ieeeAddr].publish();
     }
 
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (message.type == 'devInterview') {
-            if (!settings.getDevice(message.data)) {
-                logger.info('Connecting with device...');
-                this.mqtt.log('pairing', 'connecting with device');
-            }
+    // if debounce_ignore are specified (Array of strings)
+    // then all newPayload values with key present in debounce_ignore
+    // should equal or be undefined in oldPayload
+    // otherwise payload is conflicted
+    isPayloadConflicted(newPayload, oldPayload, debounceIgnore) {
+        let result = false;
+        Object.keys(oldPayload)
+            .filter((key) => (debounceIgnore || []).includes(key))
+            .forEach((key) => {
+                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
+                    result = true;
+                }
+            });
 
-            return;
+        return result;
+    }
+
+    canHandleEvent(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
+            return false;
         }
 
-        if (message.type == 'devIncoming') {
-            logger.info('Device incoming...');
-            this.mqtt.log('pairing', 'device incoming');
-        }
-
-        if (!device) {
-            logger.warn('Message without device!');
-            return;
-        }
-
-        if (device.ieeeAddr === this.coordinator) {
+        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
             logger.debug('Ignoring message from coordinator');
-            return;
-        }
-
-        // Check if this is a new device.
-        const settingsDevice = settings.getDevice(device.ieeeAddr);
-        if (!settingsDevice) {
-            logger.info(`New device '${device.modelId}' with address ${device.ieeeAddr} connected!`);
-            settings.addDevice(device.ieeeAddr);
-            this.mqtt.log('device_connected', device.ieeeAddr, {modelID: device.modelId});
-        }
-
-        if (!mappedDevice) {
-            logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-            return;
-        }
-
-        // After this point we cant handle message withoud cid or cmdId anymore.
-        if (!message.data || (!message.data.cid && !message.data.cmdId)) {
-            return;
+            return false;
         }
 
         /**
@@ -93,58 +77,48 @@ class DeviceReceive {
          * As the same message is also received directly from the end device, it makes no sense
          * to handle these messages.
          */
-        const hasGroupID = message.hasOwnProperty('groupid') && message.groupid != 0;
-        if (utils.isXiaomiDevice(device) && utils.isRouter(device) && hasGroupID) {
+        const hasGroupID = data.hasOwnProperty('groupID') && data.groupID != 0;
+        if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
             logger.debug('Skipping re-transmitted Xiaomi message');
+            return false;
+        }
+
+        if (!data.device.modelID && data.device.interviewing) {
+            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
+            return false;
+        }
+
+        if (!resolvedEntity.definition) {
+            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
+            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            return false;
+        }
+
+        return true;
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (!this.canHandleEvent(type, data, resolvedEntity)) {
             return;
         }
 
-        // Find a conveter for this message.
-        const cid = message.data.cid;
-        const cmdId = message.data.cmdId;
-        const converters = mappedDevice.fromZigbee.filter((c) => {
-            if (cid) {
-                // readRsp messages have the same structure as attReport messages.
-                // search for attReport converters on readRsp.
-                if (c.cid === cid) {
-                    if (c.type instanceof Array) {
-                        return c.type.includes(message.type);
-                    } else {
-                        return c.type === message.type;
-                    }
-                }
-
-                return false;
-            } else if (cmdId) {
-                return c.cmd === cmdId;
-            }
-
-            return false;
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
+            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
+            return c.cluster === data.cluster && type;
         });
 
         // Check if there is an available converter
         if (!converters.length) {
-            if (cid) {
-                // Don't log readRsp messages, they are not interesting most of the time.
-                if (message.type === 'readRsp') {
-                    return;
-                }
-
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cid '${cid}', ` +
-                    `type '${message.type}' and data '${JSON.stringify(message.data)}'`
-                );
-            } else if (cmdId) {
-                logger.warn(
-                    `No converter available for '${mappedDevice.model}' with cmd '${cmdId}' ` +
-                    `and data '${JSON.stringify(message.data)}'`
-                );
-            }
-
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            logger.debug(
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
+                `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
+            );
             return;
         }
 
+        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
+        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
+
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
         // - If a payload is returned publish it to the MQTT broker
@@ -152,45 +126,47 @@ class DeviceReceive {
         //   for e.g. click switches where we need to count number of clicks and detect long presses.
         const publish = (payload) => {
             // Add device linkquality.
-            if (message.hasOwnProperty('linkquality')) {
-                payload.linkquality = message.linkquality;
-            }
-
-            // Add last seen timestamp
-            const now = Date.now();
-            switch (settings.get().advanced.last_seen) {
-            case 'ISO_8601':
-                payload.last_seen = new Date(now).toISOString();
-                break;
-            case 'ISO_8601_local':
-                payload.last_seen = utils.toLocalISOString(new Date(now));
-                break;
-            case 'epoch':
-                payload.last_seen = now;
-                break;
+            if (data.hasOwnProperty('linkquality')) {
+                payload.linkquality = data.linkquality;
             }
 
             if (settings.get().advanced.elapsed) {
-                if (this.elapsed[device.ieeeAddr]) {
-                    payload.elapsed = now - this.elapsed[device.ieeeAddr];
+                const now = Date.now();
+                if (this.elapsed[data.device.ieeeAddr]) {
+                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
                 }
 
-                this.elapsed[device.ieeeAddr] = now;
+                this.elapsed[data.device.ieeeAddr] = now;
             }
 
             // Check if we have to debounce
-            if (settingsDevice && settingsDevice.hasOwnProperty('debounce')) {
-                this.publishDebounce(device.ieeeAddr, payload, settingsDevice.debounce);
+            if (debounce) {
+                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
             } else {
-                this.publishEntityState(device.ieeeAddr, payload);
+                this.publishEntityState(data.device.ieeeAddr, payload);
+
+                if (settings.get().homeassistant && settings.get().advanced.homeassistant_legacy_triggers) {
+                    /**
+                     * Publish an empty value for click and action payload, in this way Home Assistant
+                     * can use Home Assistant entities in automations.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
+                     */
+                    Object.keys(payload).forEach((key) => {
+                        if (['action', 'click'].includes(key)) {
+                            const counterPayload = {};
+                            counterPayload[key] = '';
+                            this.publishEntityState(data.device.ieeeAddr, counterPayload);
+                        }
+                    });
+                }
             }
         };
 
+        const meta = {device: data.device};
         let payload = {};
         converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(device.ieeeAddr)};
-            const converted = converter.convert(mappedDevice, message, publish, options);
-
+            const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
+            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
             if (converted) {
                 payload = {...payload, ...converted};
             }

lib/extension/deviceReport.js

@@ -1,105 +1,253 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const logger = require('../util/logger');
-const CC2530Router = zigbeeShepherdConverters.devices.find((d) => d.model === 'CC2530.ROUTER');
+const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
+const utils = require('../util/utils');
+const BaseExtension = require('./baseExtension');
+const debounce = require('debounce');
+const ZigbeeHerdsman = require('zigbee-herdsman');
 
-const candidates = {
-    'genOnOff': {
-        attrs: ['onOff'],
-        reportIntervalMin: 3,
-        reportIntervalMax: 300,
-        reportableChange: 0,
-    },
-    'genLevelCtrl': {
-        attrs: ['currentLevel'],
-    },
-    'lightingColorCtrl': {
-        attrs: ['colorTemperature', 'currentX', 'currentY'],
-    },
+const defaultConfiguration = {
+    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
 };
 
-const reportInterval = {
-    min: 3,
-    max: 3600,
-};
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+];
 
-const reportableChange = 1;
+const reportKey = 1;
 
-class DeviceReport {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
     }
 
-    setupReporting(mappedDevice, device) {
-        let epId = null;
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
 
-        if (mappedDevice === CC2530Router) {
-            logger.debug('Not setting up reporting for CC2530 router');
-            return;
+const clusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // Key is used this.pollDebouncers uniqueness
+        key: 1,
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerID
+        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+    },
+    {
+        key: 2,
+        cluster: {
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
+    },
+];
+
+class DeviceReport extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.configuring = new Set();
+        this.failed = new Set();
+        this.pollDebouncers = {};
+    }
+
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
         }
 
-        // Check if this device uses a different epId.
-        if (mappedDevice.hasOwnProperty('ep')) {
-            const eps = mappedDevice.ep(device);
-            epId = eps[''] || null;
-        }
+        return false;
+    }
 
-        const endpoint = this.zigbee.getEndpoint(device.ieeeAddr, epId);
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
 
-        if (!endpoint) {
-            logger.error(`Failed to setup reporting for ${device.ieeeAddr}, endpoint not found`);
-            return;
-        }
+        if (this.configuring.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.configuring.add(device.ieeeAddr);
 
-        logger.debug(`Setting up reporting for ${device.ieeeAddr}`);
-        Object.values(endpoint.clusters).filter((c) => c).forEach((c) => {
-            const cluster = c.attrs.cid;
-            if (candidates[cluster]) {
-                const candidate = candidates[cluster];
-                let attributeNames = candidate.attrs.filter((a) => c.attrs.hasOwnProperty(a));
+        try {
+            for (const ep of device.endpoints) {
+                for (const [cluster, configuration] of Object.entries(clusters)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
+                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
 
-                // Sometimes a cluster has no attributes, in this case setup reporting for all attributes.
-                attributeNames = attributeNames.length ? attributeNames : candidate.attrs;
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                items.push({...entry});
+                                delete items[items.length - 1].condition;
+                            }
+                        }
 
-                const attributes = attributeNames.map((attribute) => {
-                    return {
-                        attr: attribute,
-                        min: candidate.hasOwnProperty('reportIntervalMin')?
-                            candidate.reportIntervalMin:reportInterval.min,
-                        max: candidate.hasOwnProperty('reportIntervalMax')?
-                            candidate.reportIntervalMax:reportInterval.max,
-                        change: candidate.hasOwnProperty('reportableChange')?
-                            candidate.reportableChange:reportableChange,
-                    };
-                });
-
-                if (attributes.length > 0) {
-                    this.zigbee.report(endpoint, cluster, attributes);
+                        await ep.bind(cluster, this.coordinatorEndpoint);
+                        await ep.configureReporting(cluster, items);
+                        logger.info(
+                            `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                        );
+                    }
                 }
             }
-        });
+
+            // eslint-disable-next-line
+            device.meta.reporting = reportKey;
+        } catch (error) {
+            logger.error(
+                `Failed to setup reporting for '${device.ieeeAddr}' - ${error.stack}`,
+            );
+
+            this.failed.add(device.ieeeAddr);
+        }
+
+        device.save();
+        this.configuring.delete(device.ieeeAddr);
     }
 
-    onZigbeeStarted() {
-        this.zigbee.getAllClients().forEach((device) => {
-            const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
 
-            if (mappedDevice) {
-                this.setupReporting(mappedDevice, device);
-            }
-        });
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
+        const {device, definition} = resolvedEntity;
         // Handle messages of type endDeviceAnnce and devIncoming.
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
         // Re-setup reporting.
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (device && mappedDevice && ['endDeviceAnnce', 'devIncoming'].includes(message.type)) {
-            this.setupReporting(mappedDevice, device);
+        if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
+
+        if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
+        if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
+        // Gledopto devices don't support reporting.
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+        return true;
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                this.setupReporting(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            this.setupReporting(resolvedEntity);
+        }
+
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
+                        continue;
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${poll.key}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
         }
     }
 }

lib/extension/entityPublish.js

@@ -0,0 +1,240 @@
+
+const settings = require('../util/settings');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const assert = require('assert');
+const BaseExtension = require('./baseExtension');
+
+const postfixes = utils.getEndpointNames();
+const topicRegex = new RegExp(`^(.+?)(?:/(${postfixes.join('|')}))?/(get|set)(?:/(.+))?`);
+
+const groupConverters = [
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
+    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
+    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
+    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+];
+
+class EntityPublish extends BaseExtension {
+    onMQTTConnected() {
+        // Subscribe to topics.
+        const baseTopic = settings.get().mqtt.base_topic;
+        for (let step = 1; step < 20; step++) {
+            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
+            this.mqtt.subscribe(`${topic}set`);
+            this.mqtt.subscribe(`${topic}set/+`);
+            this.mqtt.subscribe(`${topic}get`);
+            this.mqtt.subscribe(`${topic}get/+`);
+        }
+    }
+
+    parseTopic(topic) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, postfix: match[2] || '', type: match[3], attribute: match[4]};
+    }
+
+    async onMQTTMessage(topic, message) {
+        topic = this.parseTopic(topic);
+        if (!topic) {
+            return false;
+        }
+
+        const entityName = `${topic.ID}` + (topic.postfix ? `/${topic.postfix}` : '');
+        const entity = this.zigbee.resolveEntity(entityName);
+
+        if (!entity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityName};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityName}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        let converters = null;
+        let target = null;
+        let options = {};
+        let device = null;
+        let mapped = null;
+
+        assert(entity.type === 'device' || entity.type === 'group');
+        if (entity.type === 'device') {
+            // Map device to a model
+            if (!entity.definition) {
+                logger.warn(`Device with modelID '${entity.device.modelID}' is not supported.`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
+                return;
+            }
+
+            device = entity.device;
+            mapped = entity.definition;
+            target = entity.endpoint;
+            converters = entity.definition.toZigbee;
+            options = entity.settings;
+        } else {
+            converters = groupConverters;
+            target = entity.group;
+            options = entity.settings;
+            mapped = entity.group.members.map((e) => zigbeeHerdsmanConverters.findByZigbeeModel(e.getDevice().modelID));
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        let json = {};
+        if (topic.hasOwnProperty('attribute') && topic.attribute) {
+            json[topic.attribute] = message;
+        } else {
+            try {
+                json = JSON.parse(message);
+            } catch (e) {
+                // Cannot be parsed to JSON, assume state message.
+                json = {state: message};
+            }
+        }
+
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        const deviceState = this.state.get(entity.settings.ID) || {};
+        if (settings.get().homeassistant) {
+            const hasColorTemp = json.hasOwnProperty('color_temp');
+            const hasColor = json.hasOwnProperty('color');
+            const hasBrightness = json.hasOwnProperty('brightness');
+            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete json.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+
+        // Ensure that state and brightness are executed before other commands.
+        const entries = Object.entries(json);
+        entries.sort((a, b) => (['state', 'brightness'].includes(a[0]) ? -1 : 1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters = [];
+        for (let [key, value] of entries) {
+            let postfix = topic.postfix;
+            let actualTarget = target;
+
+            // When the key has a postfix included (e.g. state_right), this will override the target.
+            if (entity.type === 'device' && key.includes('_')) {
+                const underscoreIndex = key.lastIndexOf('_');
+                const possiblePostfix = key.substring(underscoreIndex + 1, key.length);
+                if (utils.getEndpointNames().includes(possiblePostfix)) {
+                    postfix = possiblePostfix;
+                    key = key.substring(0, underscoreIndex);
+                    const device = target.getDevice();
+                    actualTarget = device.getEndpoint(mapped.endpoint(device)[postfix]);
+                }
+            }
+
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (usedConverters.includes(converter)) {
+                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${json[key]})`);
+                continue;
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {
+                endpoint_name: postfix,
+                options,
+                message: json,
+                logger,
+                device,
+                state: deviceState,
+                mapped,
+            };
+
+            try {
+                if (topic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${entity.name}'`);
+                    const result = await converter.convertSet(actualTarget, key, value, meta);
+                    if (result && result.state) {
+                        const msg = result.state;
+
+                        if (postfix) {
+                            for (const key of ['state', 'brightness']) {
+                                if (msg.hasOwnProperty(key)) {
+                                    msg[`${key}_${postfix}`] = msg[key];
+                                    delete msg[key];
+                                }
+                            }
+                        }
+
+                        this.publishEntityState(entity.settings.ID, msg);
+                    }
+
+                    // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+                    // So here we re-read the value after a specified timeout, this timeout could for example be the
+                    // transition time of a color change or for forcing a state read for devices that don't
+                    // automatically report a new state when set.
+                    // When reporting is requested for a device (report: true in device-specific settings) we won't
+                    // ever issue a read here, as we assume the device will properly report changes.
+                    // Only do this when the retrieve_state option is enabled for this device.
+                    if (
+                        entity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        entity.settings.retrieve_state
+                    ) {
+                        setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
+                    }
+                } else if (topic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${entity.name}'`);
+                    await converter.convertGet(actualTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${topic.type}' '${key}' to '${entity.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: entity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
+            }
+
+            usedConverters.push(converter);
+        }
+
+        return true;
+    }
+}
+
+module.exports = EntityPublish;

lib/extension/groups.js

@@ -1,102 +1,245 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
+const BaseExtension = require('./baseExtension');
+const utils = require('../util/utils');
+const postfixes = utils.getEndpointNames();
 
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/.+/(remove|add|remove_all)$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const topicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
-class Groups {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
+class Groups extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onStateChange = this.onStateChange.bind(this);
     }
 
     onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/add`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/+/remove_all`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
+
+        for (let step = 1; step < 20; step++) {
+            const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
+            this.mqtt.subscribe(`${topic}remove`);
+            this.mqtt.subscribe(`${topic}add`);
+            this.mqtt.subscribe(`${topic}remove_all`); // DEPRECATED
+        }
     }
 
-    parseTopic(topic) {
-        if (!topic.match(topicRegex)) {
-            return null;
-        }
-
-        // Remove base from topic
-        topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/group/`, '');
-
-        // Parse type from topic
-        const type = topic.substr(topic.lastIndexOf('/') + 1, topic.length);
-
-        // Remove type from topic
-        topic = topic.replace(`/${type}`, '');
-
-        return {friendly_name: topic, type};
+    async onZigbeeStarted() {
+        await this.syncGroupsWithSettings();
+        this.eventBus.on('stateChange', this.onStateChange);
     }
 
-    onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
+    async syncGroupsWithSettings() {
+        const settingsGroups = settings.getGroups();
+        const zigbeeGroups = this.zigbee.getGroups();
+        for (const settingGroup of settingsGroups) {
+            const groupID = settingGroup.ID;
+            const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
+            const settingsEntity = settingGroup.devices.map((d) => {
+                const entity = this.zigbee.resolveEntity(d);
+                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
+                return entity;
+            }).filter((e) => e != null);
 
-        if (!topic) {
-            return false;
+            // In settings but not in zigbee
+            for (const entity of settingsEntity) {
+                if (!zigbeeGroup.hasMember(entity.endpoint)) {
+                    logger.info(`Adding '${entity.name}' to group '${settingGroup.friendlyName}'`);
+                    await entity.endpoint.addToGroup(zigbeeGroup);
+                }
+            }
+
+            // In zigbee but not in settings
+            for (const endpoint of zigbeeGroup.members) {
+                if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
+                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
+                    logger.info(`Removing '${deviceSettings.friendlyName}' from group '${settingGroup.friendlyName}'`);
+                    await endpoint.removeFromGroup(zigbeeGroup);
+                }
+            }
         }
 
-        // Find ID of this group.
-        const groupID = settings.getGroupIDByFriendlyName(topic.friendly_name);
-        if (!groupID) {
-            logger.error(`Group with friendly_name '${topic.friendly_name}' doesn't exist`);
+        // eslint-disable-next-line
+        for (const zigbeeGroup of zigbeeGroups) {
+            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
+                for (const endpoint of zigbeeGroup.members) {
+                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
+                    logger.info(`Removing '${deviceSettings.friendlyName}' from group ${zigbeeGroup.groupID}`);
+                    await endpoint.removeFromGroup(zigbeeGroup);
+                }
+            }
+        }
+    }
+
+    async onStateChange(data) {
+        const reason = 'group_optimistic';
+        if (data.reason === reason) {
             return;
         }
 
-        if (groupID === 99) {
-            logger.error('Group 99 is reserved, please use a different groupID');
-            return;
-        }
+        const properties = ['state', 'brightness', 'color_temp', 'color'];
+        const payload = {};
 
-        // Map message to ieeeAddr and check if device exist.
-        message = message.toString();
-        const ieeeAddr = settings.getIeeeAddrByFriendlyName(message) || message;
-        if (!this.zigbee.getDevice(ieeeAddr)) {
-            logger.error(`Failed to find device '${message}'`);
-            return;
-        }
+        properties.forEach((prop) => {
+            if (data.to.hasOwnProperty(prop)) {
+                payload[prop] = data.to[prop];
+            }
+        });
 
-        // Send command to the device.
-        let payload = null;
-        let cmd = null;
-        if (topic.type === 'add') {
-            payload = {groupid: groupID, groupname: ''};
-            cmd = 'add';
-        } else if (topic.type === 'remove') {
-            payload = {groupid: groupID};
-            cmd = 'remove';
-        } else if (topic.type === 'remove_all') {
-            payload = {};
-            cmd = 'removeAll';
-        }
+        if (Object.keys(payload).length) {
+            const entity = this.zigbee.resolveEntity(data.ID);
+            const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
+                const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
+                return settingsGroup && settingsGroup.optimistic;
+            });
 
-        const callback = (error, rsp) => {
-            if (error) {
-                logger.error(`Failed to ${topic.type} ${ieeeAddr} from ${topic.friendly_name}`);
+            if (entity.type === 'device') {
+                for (const zigbeeGroup of zigbeeGroups) {
+                    if (zigbeeGroup.hasMember(entity.endpoint)) {
+                        if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                            await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
+                        }
+                    }
+                }
             } else {
-                logger.info(`Successfully ${topic.type} ${ieeeAddr} to ${topic.friendly_name}`);
+                const groupIDsToPublish = new Set();
+                for (const member of entity.group.members) {
+                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
+                    for (const zigbeeGroup of zigbeeGroups) {
+                        if (zigbeeGroup.hasMember(member)) {
+                            if (!payload || payload.state !== 'OFF' || this.allMembersOff(zigbeeGroup)) {
+                                groupIDsToPublish.add(zigbeeGroup.groupID);
+                            }
+                        }
+                    }
+                }
+                groupIDsToPublish.delete(entity.group.groupID);
+                for (const groupID of groupIDsToPublish) {
+                    await this.publishEntityState(groupID, payload, reason);
+                }
+            }
+        }
+    }
 
-                // Log to MQTT
-                const log = {device: message};
-                if (['remove', 'add'].includes(topic.type)) {
-                    log.group = topic.friendly_name;
+    allMembersOff(zigbeeGroup) {
+        for (const member of zigbeeGroup.members) {
+            const device = member.getDevice();
+            if (this.state.exists(device.ieeeAddr)) {
+                const state = this.state.get(device.ieeeAddr);
+                if (state && state.state === 'ON') {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    async onMQTTMessage(topic, message) {
+        let type;
+        let group;
+        const topicMatch = topic.match(topicRegex);
+        if (topicMatch) {
+            group = this.zigbee.resolveEntity(topicMatch[1]);
+            type = topicMatch[2];
+
+            if (!group || group.type !== 'group') {
+                logger.error(`Group '${topicMatch[1]}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
                 }
 
-                this.mqtt.log(log);
+                return;
             }
-        };
+        } else if (topic.match(topicRegexRemoveAll)) {
+            type = 'remove_all';
+        } else {
+            return;
+        }
 
-        this.zigbee.publish(
-            ieeeAddr, 'device', 'genGroups', cmd, 'functional',
-            payload, null, null, callback,
-        );
+        const entity = this.zigbee.resolveEntity(message);
+        if (!entity || !entity.type === 'device') {
+            logger.error(`Device '${message}' does not exist`);
 
-        return true;
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                );
+            }
+
+            return;
+        }
+
+        const keys = [
+            `${entity.device.ieeeAddr}/${entity.endpoint.ID}`,
+            `${entity.name}/${entity.endpoint.ID}`,
+        ];
+
+        const definition = entity.definition;
+        const endpoints = definition && definition.endpoint ? definition.endpoint(entity.device) : null;
+        const endpointName = endpoints ? Object.entries(endpoints).find((e) => e[1] === entity.endpoint.ID)[0] : null;
+
+        if (endpointName) {
+            keys.push(`${entity.device.ieeeAddr}/${endpointName}`);
+            keys.push(`${entity.name}/${endpointName}`);
+        }
+
+        const hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+        if (!hasEndpointName) {
+            keys.push(entity.name);
+            keys.push(entity.device.ieeeAddr);
+        }
+
+        if (type === 'add') {
+            logger.info(`Adding '${entity.name}' to '${group.name}'`);
+            await entity.endpoint.addToGroup(group.group);
+            settings.addDeviceToGroup(group.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: entity.name, group: group.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_add`, message: payload}),
+                );
+            }
+        } else if (type === 'remove') {
+            logger.info(`Removing '${entity.name}' from '${group.name}'`);
+            await entity.endpoint.removeFromGroup(group.group);
+            settings.removeDeviceFromGroup(group.settings.ID, keys);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const payload = {friendly_name: entity.name, group: group.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `device_group_remove`, message: payload}),
+                );
+            }
+        } else { // remove_all
+            logger.info(`Removing '${entity.name}' from all groups`);
+            await entity.endpoint.removeFromAllGroups();
+            for (const settingsGroup of settings.getGroups()) {
+                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {friendly_name: entity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
+                    );
+                }
+            }
+        }
     }
 }
 

lib/extension/livolo.js

@@ -1,183 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(1);
-const logger = require('../util/logger');
-
-const foundationCfg = {manufSpec: 0, disDefaultRsp: 0};
-
-/**
- * Extension required for Livolo device support.
- */
-class Livolo {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-        this.configured = {};
-    }
-
-    onZigbeeStarted() {
-        this.startTimer();
-    }
-
-    _resetDeviceState(ieeeAddr) {
-        this.configured[ieeeAddr] = {
-            stage: 0,
-            retry: 0,
-            waitresp: false,
-        };
-    }
-
-    onZigbeeMessage(message, device, mappedDevice) {
-        if (!device) {
-            return;
-        }
-
-        if ((message.type == 'devInterview') ||
-            (message.type == 'devIncoming') ||
-            (message.type == 'endDeviceAnnce') ||
-            (message.type == 'devStatus')) {
-            if (this.configured.hasOwnProperty(device.ieeeAddr)) {
-                logger.info(`LIVOLO ${device.ieeeAddr}. (Re)joins in the network (after power off?)`);
-                this._resetDeviceState(device.ieeeAddr);
-            }
-            return;
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    _handleCommandRespSimple(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            if (this.ctype === 'toggle') {
-                logger.debug(`LIVOLO ${this.ieeeAddr}. Toggle command error:`, err.message);
-            }
-        } else {
-            logger.debug(`LIVOLO ${this.ieeeAddr}. Sucessfully configured`, rsp);
-            this.ext.configured[this.ieeeAddr].stage = 1; // sucessfully send command
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-        }
-    }
-
-    _handleCommandRespWithData(err, rsp) {
-        this.ext.configured[this.ieeeAddr].waitresp = false;
-
-        if (err) {
-            logger.info(`LIVOLO ${this.ieeeAddr}. ${this.cid}.${this.ctype} response error:`, err.message);
-            if (this.ext.configured[this.ieeeAddr].retry >= 3) {
-                // errors in three sequental reads, stop polling, wait for a device message
-                this.device.status = 'offline';
-                logger.info(`LIVOLO ${this.ieeeAddr}. Stopped polling after 3 unsuccessful attempts`);
-            }
-        } else {
-            this.ext.configured[this.ieeeAddr].retry = 0;
-            this.device.status = 'online';
-            if (this.ext.zigbee) {
-                this.ext.zigbee.shepherd.emit('ind:reported', this.ep, this.cid, rsp, this.ep.last_af_msg);
-            }
-        }
-    }
-
-    // msg: { groupid, clusterid, srcaddr, srcendpoint, dstendpoint, wasbroadcast,
-    //   linkquality, securityuse, timestamp, transseqnumber, len, data }
-    _handleAfMessage(msg, ep) {
-        ep.linkquality = msg.linkquality;
-        ep.last_af_msg = msg;
-    }
-
-    _sendToggle(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            const cfg = {};
-            logger.debug(`LIVOLO ${ieeeAddr}. Sending the 'toggle' command. Retry: ${retry}`);
-            ep.functional('genOnOff', 'toggle', [cfg], foundationCfg,
-                this._handleCommandRespSimple.bind({
-                    device: zdev,
-                    ieeeAddr,
-                    cid: 'genOnOff',
-                    ctype: 'toggle',
-                    ext: this,
-                }));
-
-            queueCallback();
-        });
-    }
-
-    _sendPoll(zdev, ieeeAddr, ep, retry) {
-        this.zigbee.queue.push(ieeeAddr, (queueCallback) => {
-            ep.foundation('genOnOff', 'read', [{
-                attrId: 0, // onOff
-            }], this._handleCommandRespWithData.bind({
-                device: zdev,
-                ieeeAddr,
-                ep,
-                cid: 'genOnOff',
-                ctype: 'read',
-                ext: this,
-            }));
-
-            queueCallback();
-        });
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => d.manufName && d.manufName.startsWith('LIVOLO')) // LIVOLO
-            .filter((d) => d.type === 'EndDevice') // Filter end devices
-            .filter((d) => d.powerSource && d.powerSource !== 'Battery') // Remove battery powered devices
-            .forEach((d) => {
-                const zdev = this.zigbee.shepherd._findDevByAddr(d.ieeeAddr);
-                if (zdev && zdev.endpoints) {
-                    const eplist = Object.keys(zdev.endpoints).filter((epId) => {
-                        const ep2 = zdev.getEndpoint(epId);
-                        const clist = ep2.getClusterList();
-                        return clist && clist.includes(6); // 6 - genOnOff
-                    });
-
-                    if (eplist.length > 0) {
-                        const ep = zdev.getEndpoint(eplist[0]);
-
-                        ep.onAfIncomingMsg = this._handleAfMessage;
-
-                        if (!this.configured.hasOwnProperty(d.ieeeAddr)) {
-                            this._resetDeviceState(d.ieeeAddr);
-                        }
-
-                        const state = this.configured[d.ieeeAddr];
-                        if (state.waitresp) {
-                            return;
-                        }
-
-                        if (state.retry < 3) {
-                            if (state.stage === 0) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendToggle(zdev, d.ieeeAddr, ep, state.retry);
-                            } else if (state.stage === 1) {
-                                state.retry += 1;
-                                state.waitresp = true;
-                                this._sendPoll(zdev, d.ieeeAddr, ep, state.retry);
-                            }
-                        }
-                    }
-                }
-            });
-        return true;
-    }
-}
-
-module.exports = Livolo;

lib/extension/networkMap.js

@@ -1,14 +1,20 @@
 const settings = require('../util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
+const utils = require('../util/utils');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../util/logger');
+const BaseExtension = require('./baseExtension');
 
-class NetworkMap {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
+class NetworkMap extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         // Subscribe to topic.
         this.topic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.topicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+
+        // Bind
+        this.raw = this.raw.bind(this);
+        this.graphviz = this.graphviz.bind(this);
 
         // Set supported formats
         this.supportedFormats = {
@@ -19,75 +25,90 @@ class NetworkMap {
 
     onMQTTConnected() {
         this.mqtt.subscribe(this.topic);
+        this.mqtt.subscribe(this.topicRoutes);
     }
 
-    onMQTTMessage(topic, message) {
-        message = message.toString();
-
-        if (topic === this.topic && this.supportedFormats.hasOwnProperty(message)) {
-            this.zigbee.networkScan((result)=> {
-                const converted = this.supportedFormats[message](this.zigbee, result);
-                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
-            });
-
-            return true;
+    async onMQTTMessage(topic, message) {
+        if ((topic === this.topic || topic === this.topicRoutes) && this.supportedFormats.hasOwnProperty(message)) {
+            const includeRoutes = topic === this.topicRoutes;
+            const topology = await this.networkScan(includeRoutes);
+            const converted = this.supportedFormats[message](topology);
+            this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
         }
-
-        return false;
     }
 
-    raw(zigbee, topology) {
+    raw(topology) {
         return JSON.stringify(topology);
     }
 
-    graphviz(zigbee, topology) {
-        let text = 'digraph G {\nnode[shape=record];\n';
-        let devStyle = '';
+    graphviz(topology) {
+        const colors = settings.get().map_options.graphviz.colors;
 
-        zigbee.getDevices().forEach((device) => {
+        let text = 'digraph G {\nnode[shape=record];\n';
+        let style = '';
+
+        topology.nodes.forEach((device) => {
             const labels = [];
-            const friendlyDevice = settings.getDevice(device.ieeeAddr);
-            const friendlyName = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
 
             // Add friendly name
-            labels.push(friendlyName);
+            labels.push(`${device.friendlyName}`);
 
-            // Add the device type
-            labels.push(device.type);
+            // Add the device short network address, ieeaddr and scan note (if any)
+            labels.push(
+                `${device.ieeeAddr} (${device.networkAddress})` +
+                ((device.failed && device.failed.length) ? `failed: ${device.failed.join(',')}` : ''),
+            );
 
             // Add the device model
-            const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-            if (mappedModel) {
-                labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
-            } else {
-                // This model is not supported by zigbee-shepherd-converters, add zigbee model information, if available
-                const zigbeeModel = [device.manufName, device.modelId].filter((a) => a).join(' ');
-                labels.push(zigbeeModel ? zigbeeModel : 'No model information available');
+            if (device.type !== 'Coordinator') {
+                const mappedModel = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                if (mappedModel) {
+                    labels.push(`${mappedModel.vendor} ${mappedModel.description} (${mappedModel.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    labels.push(`${device.manufacturerName} ${device.modelID}`);
+                }
             }
 
-            // Add the device status (online/offline)
-            labels.push(device.status);
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = device.type === 'Coordinator' ? Date.now() : device.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+            }
+
+            labels.push(lastSeen);
 
             // Shape the record according to device type
             if (device.type == 'Coordinator') {
-                devStyle = 'style="bold"';
+                style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
+                    `fontcolor="${colors.font.coordinator}"`;
             } else if (device.type == 'Router') {
-                devStyle = 'style="rounded"';
+                style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
+                    `fontcolor="${colors.font.router}"`;
             } else {
-                devStyle = 'style="rounded, dashed"';
+                style = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", ` +
+                    `fontcolor="${colors.font.enddevice}"`;
             }
 
             // Add the device with its labels to the graph as a node.
-            text += `  "${device.ieeeAddr}" [`+devStyle+`, label="{${labels.join('|')}}"];\n`;
+            text += `  "${device.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
 
             /**
-             * Add an edge between the device and its parent to the graph
+             * Add an edge between the device and its child to the graph
              * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
              * due to not responded to the lqi scan. In that case we do not add an edge for this device.
              */
-            topology.filter((e) => e.ieeeAddr === device.ieeeAddr).forEach((e) => {
-                const lineStyle = (e.lqi==0) ? `style="dashed", ` : ``;
-                text += `  "${device.ieeeAddr}" -> "${e.parent}" [`+lineStyle+`label="${e.lqi}"]\n`;
+            topology.links.filter((e) => (e.source.ieeeAddr === device.ieeeAddr)).forEach((e) => {
+                const lineStyle = (device.type=='EndDevice') ? 'penwidth=1, ' :
+                    (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
+                const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
+                    `weight=1, color="${colors.line.active}", `;
+                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
+                    `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
+                text += `  "${device.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
             });
         });
 
@@ -95,6 +116,81 @@ class NetworkMap {
 
         return text.replace(/\0/g, '');
     }
+
+    async networkScan(includeRoutes) {
+        logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
+        const devices = this.zigbee.getDevices();
+        const lqis = new Map();
+        const routingTables = new Map();
+        const failed = new Map();
+
+        for (const device of devices.filter((d) => d.type != 'EndDevice')) {
+            failed.set(device, []);
+            const resolved = this.zigbee.resolveEntity(device);
+            try {
+                const result = await device.lqi();
+                lqis.set(device, result);
+                logger.debug(`LQI succeeded for '${resolved.name}'`);
+            } catch (error) {
+                failed.get(device).push('lqi');
+                logger.error(`Failed to execute LQI for '${resolved.name}'`);
+            }
+
+            if (includeRoutes) {
+                try {
+                    const result = await device.routingTable();
+                    routingTables.set(device, result);
+                    logger.debug(`Routing table succeeded for '${resolved.name}'`);
+                } catch (error) {
+                    failed.get(device).push('routingTable');
+                    logger.error(`Failed to execute routing table for '${resolved.name}'`);
+                }
+            }
+        }
+
+        logger.info(`Network scan finished`);
+
+        const networkMap = {nodes: [], links: []};
+        // Add nodes
+        for (const device of devices) {
+            const resolved = this.zigbee.resolveEntity(device);
+            networkMap.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: resolved.name, type: device.type,
+                networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
+                modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+            });
+        }
+
+        // Add links
+        lqis.forEach((lqi, device) => {
+            for (const neighbor of lqi.neighbors) {
+                if (neighbor.relationship > 3) {
+                    // Relationship is not active, skip it
+                    continue;
+                }
+
+                const link = {
+                    source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
+                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
+                    linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
+                    // DEPRECATED:
+                    sourceIeeeAddr: neighbor.ieeeAddr, targetIeeeAddr: device.ieeeAddr,
+                    sourceNwkAddr: neighbor.networkAddress, lqi: neighbor.linkquality,
+                    relationship: neighbor.relationship,
+                };
+
+                const routingTable = routingTables.get(device);
+                if (routingTable) {
+                    link.routes = routingTable.table
+                        .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
+                }
+
+                networkMap.links.push(link);
+            }
+        });
+
+        return networkMap;
+    }
 }
 
 module.exports = NetworkMap;

lib/extension/otaUpdate.js

@@ -0,0 +1,206 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const assert = require('assert');
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const BaseExtension = require('./baseExtension');
+const MINUTES_10 = 1000 * 60 * 10;
+
+class OTAUpdate extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.inProgress = new Set();
+        this.lastChecked = {};
+    }
+
+    onMQTTConnected() {
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
+        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        if (supportsOTA) {
+            // When a device does a next image request, it will usually do it a few times after each other
+            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
+            // each time.
+            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+
+            this.lastChecked[data.device.ieeeAddr] = Date.now();
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            this.publishEntityState(data.device.ieeeAddr, {update_available: available});
+
+            if (available) {
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        }
+
+        // Respond to the OTA request:
+        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
+        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+    }
+
+    async readSoftwareBuildIDAndDateCode(device, update) {
+        try {
+            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
+
+            if (update) {
+                device.softwareBuildID = result.swBuildId;
+                device.dateCode = result.dateCode;
+                device.save();
+            }
+
+            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
+        } catch (e) {
+            return null;
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (!topic.match(topicRegex)) {
+            return null;
+        }
+
+        const device = this.zigbee.resolveEntity(message);
+        assert(device != null && device.type === 'device', 'Device not found or not a device');
+        if (!device.definition || !device.definition.ota) {
+            const message = `Device '${device.name}' does not support OTA updates`;
+            logger.error(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: device.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            return;
+        }
+
+        if (this.inProgress.has(device.device.ieeeAddr)) {
+            logger.error(`Update or check already in progress for '${device.name}', skipping...`);
+            return;
+        }
+        this.inProgress.add(device.device.ieeeAddr);
+
+        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
+        if (type === 'check') {
+            const message = `Checking if update available for '${device.name}'`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `checking_if_available`, device: device.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const available = await device.definition.ota.isUpdateAvailable(device.device, logger);
+                const message=(available ?
+                    `Update available for '${device.name}'` : `No update available for '${device.name}'`);
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: available ? 'available' : 'not_available', device: device.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+
+                this.publishEntityState(device.device.ieeeAddr, {update_available: available});
+                this.lastChecked[device.device.ieeeAddr] = Date.now();
+            } catch (error) {
+                const message = `Failed to check if update available for '${device.name}' (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `check_failed`, device: device.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        JSON.stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        } else { // type === 'update'
+            const message = `Updating '${device.name}' to latest firmware`;
+            logger.info(message);
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `update_in_progress`, device: device.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    JSON.stringify({type: `ota_update`, message, meta}),
+                );
+            }
+
+            try {
+                const onProgress = (progress, remaining) => {
+                    let message = `Update of '${device.name}' at ${progress}%`;
+                    if (remaining) {
+                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                    }
+
+                    logger.info(message);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_progress`, device: device.name, progress};
+                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    }
+                };
+
+                const from_ = await this.readSoftwareBuildIDAndDateCode(device.device, false);
+                await device.definition.ota.updateToLatest(device.device, logger, onProgress);
+                const to = await this.readSoftwareBuildIDAndDateCode(device.device, true);
+                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
+                const message = `Finished update of '${device.name}'` + (to ? `, from '${fromS}' to '${toS}'` : ``);
+                logger.info(message);
+                this.publishEntityState(device.device.ieeeAddr, {update_available: false});
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            } catch (error) {
+                const message = `Update of '${device.name}' failed (${error.message})`;
+                logger.error(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_failed`, device: device.name};
+                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                }
+            }
+        }
+
+        this.inProgress.delete(device.device.ieeeAddr);
+    }
+}
+
+module.exports = OTAUpdate;

lib/extension/softReset.js

@@ -1,13 +1,16 @@
+/* istanbul ignore file */
+// DEPRECATED
 const settings = require('../util/settings');
 const logger = require('../util/logger');
 const utils = require('../util/utils');
+const BaseExtension = require('./baseExtension');
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
+class SoftReset extends BaseExtension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;
         this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
     }
@@ -33,29 +36,29 @@ class SoftReset {
         this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
     }
 
-    handleTimeout() {
+    async handleTimeout() {
         logger.warn('Soft reset timeout triggered');
 
-        this.zigbee.softReset((error) => {
-            if (error) {
-                logger.warn('Soft reset failed, trying stop/start');
-                this.zigbee.stop((error) => {
-                    logger.warn('Zigbee stopped');
-                    this.zigbee.start((error) => {
-                        if (error) {
-                            logger.error('Failed to restart!');
-                        }
-                    });
-                });
-            } else {
-                logger.warn('Soft resetted ZNP due to timeout');
-            }
+        try {
+            await this.zigbee.reset('soft');
+            logger.warn('Soft resetted ZNP due to timeout');
+        } catch (error) {
+            logger.warn('Soft reset failed, trying stop/start');
 
-            this.resetTimer();
-        });
+            await this.zigbee.stop();
+            logger.warn('Zigbee stopped');
+
+            try {
+                await this.zigbee.start();
+            } catch (error) {
+                logger.error('Failed to restart!');
+            }
+        }
+
+        this.resetTimer();
     }
 
-    onZigbeeMessage(message, device, mappedDevice) {
+    onZigbeeEvent() {
         this.resetTimer();
     }
 }

lib/extension/xiaomi.js

@@ -1,70 +0,0 @@
-const utils = require('../util/utils');
-const interval = utils.secondsToMilliseconds(60);
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-
-/**
- * This extensions handles Xiaomi devices.
- * - Marks Xiaomi devices as online.
- * - Polls Xiaomi routers to keep them awake.
- */
-class Xiaomi {
-    constructor(zigbee, mqtt, state, publishEntityState) {
-        this.zigbee = zigbee;
-        this.timer = null;
-    }
-
-    onZigbeeStarted() {
-        // Set all Xiaomi devices to be online, so shepherd won't try
-        // to query info from devices (which would fail because they go to sleep).
-        const devices = this.zigbee.getAllClients();
-        devices.forEach((d) => {
-            if (utils.isXiaomiDevice(d)) {
-                const device = this.zigbee.shepherd.find(d.ieeeAddr, 1);
-                if (device) {
-                    device.getDevice().update({
-                        status: 'online',
-                        joinTime: Math.floor(Date.now() / 1000),
-                    });
-                }
-            }
-        });
-
-        /**
-         * Device availability already pings Xiaomi devices.
-         */
-        if (!settings.get().advanced.availability_timeout) {
-            this.startTimer();
-        } else {
-            logger.debug('Disabled pinging of Xiaomi devices in Xiaomi extension because availability is enabled');
-        }
-    }
-
-    startTimer() {
-        this.clearTimer();
-        this.timer = setInterval(() => this.handleInterval(), interval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-    }
-
-    ping(ieeeAddr) {
-        this.zigbee.ping(ieeeAddr, 'error', null, 'basic');
-    }
-
-    handleInterval() {
-        this.zigbee.getAllClients()
-            .filter((d) => utils.isXiaomiDevice(d) && utils.isRouter(d) && !utils.isBatteryPowered(d))
-            .forEach((d) => this.ping(d.ieeeAddr));
-    }
-}
-
-module.exports = Xiaomi;

lib/mqtt.js

@@ -1,15 +1,16 @@
 const mqtt = require('mqtt');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
+const fs = require('fs');
+const events = require('events');
 
-class MQTT {
+class MQTT extends events.EventEmitter {
     constructor() {
-        this.onConnect = this.onConnect.bind(this);
+        super();
         this.onMessage = this.onMessage.bind(this);
-        this.messageHandler = null;
     }
 
-    connect(messageHandler, callback) {
+    async connect() {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
@@ -21,6 +22,27 @@ class MQTT {
             },
         };
 
+        if (mqttSettings.version) {
+            options.protocolVersion = mqttSettings.version;
+        }
+
+        if (mqttSettings.keepalive) {
+            logger.debug(`Using MQTT keepalive: ${mqttSettings.keepalive}`);
+            options.keepalive = mqttSettings.keepalive;
+        }
+
+        if (mqttSettings.ca) {
+            logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
+            options.ca = fs.readFileSync(mqttSettings.ca);
+        }
+
+        if (mqttSettings.key && mqttSettings.cert) {
+            logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
+            logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
+            options.key = fs.readFileSync(mqttSettings.key);
+            options.cert = fs.readFileSync(mqttSettings.cert);
+        }
+
         if (mqttSettings.user && mqttSettings.password) {
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
@@ -36,16 +58,6 @@ class MQTT {
             options.rejectUnauthorized = false;
         }
 
-        this.client = mqtt.connect(mqttSettings.server, options);
-
-        // Register callbacks.
-        this.client.on('connect', () => {
-            this.onConnect();
-            callback();
-        });
-
-        this.client.on('message', this.onMessage);
-
         // Set timer at interval to check if connected to MQTT server.
         const interval = 10 * 1000; // seconds * 1000.
         this.connectionTimer = setInterval(() => {
@@ -54,22 +66,25 @@ class MQTT {
             }
         }, interval);
 
-        this.messageHandler = messageHandler;
-    }
+        return new Promise((resolve) => {
+            this.client = mqtt.connect(mqttSettings.server, options);
 
-    disconnect() {
-        clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
+            this.client.on('connect', () => {
+                logger.info('Connected to MQTT server');
+                this.publish('bridge/state', 'online', {retain: true, qos: 0});
+                resolve();
+            });
 
-        this.publish('bridge/state', 'offline', {retain: true, qos: 0}, () => {
-            logger.info('Disconnecting from MQTT server');
-            this.client.end();
+            this.client.on('message', this.onMessage);
         });
     }
 
-    onConnect() {
-        logger.info('Connected to MQTT server');
-        this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    async disconnect() {
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = null;
+        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        logger.info('Disconnecting from MQTT server');
+        this.client.end();
     }
 
     subscribe(topic) {
@@ -77,13 +92,12 @@ class MQTT {
     }
 
     onMessage(topic, message) {
-        if (this.messageHandler) {
-            this.messageHandler(topic, message);
-        }
+        this.emit('message', {topic, message: message.toString()});
     }
 
-    publish(topic, payload, options, callback, base=settings.get().mqtt.base_topic) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
         topic = `${base}/${topic}`;
+        options = {qos: 0, retain: false, ...options};
 
         if (!this.client || this.client.reconnecting) {
             logger.error(`Not connected to MQTT server!`);
@@ -92,17 +106,10 @@ class MQTT {
         }
 
         logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
-        this.client.publish(topic, payload, options, callback);
-    }
 
-    log(type, message, meta=null) {
-        const payload = {type, message};
-
-        if (meta) {
-            payload.meta = meta;
-        }
-
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
+        return new Promise((resolve) => {
+            this.client.publish(topic, payload, options, () => resolve());
+        });
     }
 }
 

lib/state.js

@@ -1,24 +1,22 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
 const fs = require('fs');
-const settings = require('./util/settings');
 const objectAssignDeep = require('object-assign-deep');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'action_color_temperature', 'action_color',
-    'action_group', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
 ];
 
 class State {
-    constructor() {
+    constructor(eventBus) {
         this.state = {};
         this.file = data.joinPath('state.json');
         this.timer = null;
-
-        this.handleSettingsChanged = this.handleSettingsChanged.bind(this);
+        this.eventBus = eventBus;
     }
 
     start() {
@@ -27,27 +25,6 @@ class State {
         // Save the state on every interval
         this.clearTimer();
         this.timer = setInterval(() => this.save(), saveInterval);
-
-        // Listen for on settings changed events.
-        settings.addOnChangeHandler(this.handleSettingsChanged);
-
-        this.checkLastSeen();
-    }
-
-    handleSettingsChanged() {
-        this.checkLastSeen();
-    }
-
-    checkLastSeen() {
-        if (settings.get().advanced.last_seen === 'disable') {
-            Object.values(this.state).forEach((s) => {
-                if (s.hasOwnProperty('last_seen')) {
-                    delete s.last_seen;
-                }
-            });
-
-            this.save();
-        }
     }
 
     clearTimer() {
@@ -81,28 +58,33 @@ class State {
         fs.writeFileSync(this.file, json, 'utf8');
     }
 
-    exists(ieeeAddr) {
-        return this.state.hasOwnProperty(ieeeAddr);
+    exists(ID) {
+        return this.state.hasOwnProperty(ID);
     }
 
-    get(ieeeAddr) {
-        return this.state[ieeeAddr];
+    get(ID) {
+        return this.state[ID];
     }
 
-    set(ieeeAddr, state) {
-        const s = objectAssignDeep.noMutate(state);
-        dontCacheProperties.forEach((property) => {
-            if (s.hasOwnProperty(property)) {
-                delete s[property];
+    set(ID, state, reason=null) {
+        const toState = objectAssignDeep.noMutate(state);
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
+                delete toState[property];
             }
-        });
+        }
 
-        this.state[ieeeAddr] = s;
+        const fromState = this.state[ID];
+
+        this.state[ID] = toState;
+
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
     }
 
-    remove(ieeeAddr) {
-        if (this.exists(ieeeAddr)) {
-            delete this.state[ieeeAddr];
+    remove(ID) {
+        if (this.exists(ID)) {
+            delete this.state[ID];
         }
     }
 }

lib/util/fs.js

@@ -1,16 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-
-function readYaml(file) {
-    return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-}
-
-function readYamlIfExists(file) {
-    return fs.existsSync(file) ? readYaml(file) : null;
-}
-
-function writeYaml(file, content) {
-    fs.writeFileSync(file, yaml.safeDump(content));
-}
-
-module.exports = {readYaml, readYamlIfExists, writeYaml};

lib/util/logger.js

@@ -5,57 +5,116 @@ const path = require('path');
 const fs = require('fs');
 const fx = require('mkdir-recursive');
 const rimraf = require('rimraf');
+const colorizer = winston.format.colorize();
+const assert = require('assert');
+
+// What transports to enable
+const output = settings.get().advanced.log_output;
+
+// Directory to log to
+const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+// Make sure that log directoy exsists when not logging to stdout only
+if (output.includes('file')) {
+    fx.mkdirSync(directory);
+}
 
 // Determine the log level.
 const level = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
 
-// Directoy to log to
-const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
-const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const levelWithCompensatedLength = {
+    'info': 'info ',
+    'error': 'error',
+    'warn': 'warn ',
+    'debug': 'debug',
+};
 
-// Make sure that log directoy exsists
-fx.mkdirSync(directory);
+/* istanbul ignore next */
+const timestampFormat = () => moment().format(settings.get().advanced.timestamp_format);
+
+// Setup default console logger
+const transportsToUse = [
+    new winston.transports.Console({
+        level,
+        silent: !output.includes('console'),
+        format: winston.format.combine(
+            winston.format.timestamp({format: timestampFormat}),
+            winston.format.printf(/* istanbul ignore next */(info) => {
+                const {timestamp, level, message} = info;
+                const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
+                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+            }),
+        ),
+    }),
+];
+
+// Add file logger when enabled
+// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            const {timestamp, level, message} = info;
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
+if (output.includes('file')) {
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
 
 // Create logger
-const logger = new winston.Logger({
-    transports: [
-        new winston.transports.File({
-            filename: path.join(directory, 'log.txt'),
-            json: false,
-            level,
-            maxFiles: 3, // Keep last 3 files
-            maxsize: 10000000, // 10MB
-            timestamp: () => new Date().toLocaleString(),
-        }),
-        new winston.transports.Console({
-            timestamp: () => new Date().toLocaleString(),
-            formatter: (options) =>
-                winston.config.colorize(options.level, '  zigbee2mqtt:' + options.level.toLowerCase()) + ' ' +
-                options.timestamp() + ' ' + (options.message ? options.message : '') +
-                (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' ),
-        }),
-    ],
-});
-
-logger.info(`Logging to directory: '${directory}'`);
-
-logger.transports.console.level = level;
+const logger = winston.createLogger({transports: transportsToUse});
 
 // Cleanup any old log directory.
-if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
-    const rootDirectory = path.join(directory, '..');
+function cleanup() {
+    if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
+        const rootDirectory = path.join(directory, '..');
 
-    let directories = fs.readdirSync(rootDirectory).map((d) => {
-        d = path.join(rootDirectory, d);
-        return {path: d, birth: fs.statSync(d).birthtimeMs};
-    });
+        let directories = fs.readdirSync(rootDirectory).map((d) => {
+            d = path.join(rootDirectory, d);
+            return {path: d, birth: fs.statSync(d).mtime};
+        });
 
-    directories.sort((a, b) => b.birth - a.birth);
-    directories = directories.slice(10, directories.length);
-    directories.forEach((dir) => {
-        logger.debug(`Removing old log directory '${dir.path}'`);
-        rimraf.sync(dir.path);
-    });
+        directories.sort((a, b) => b.birth - a.birth);
+        directories = directories.slice(10, directories.length);
+        directories.forEach((dir) => {
+            logger.debug(`Removing old log directory '${dir.path}'`);
+            rimraf.sync(dir.path);
+        });
+    }
+}
+
+logger.cleanup = cleanup;
+logger.getLevel = () => transportsToUse[0].level;
+logger.setLevel = (level) => {
+    transportsToUse.forEach((transport) => transport.level = level);
+};
+
+// Print to user what logging is enabled
+if (output.includes('file')) {
+    if (output.includes('console')) {
+        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+    } else {
+        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+    }
+    logger.cleanup();
+} else if (output.includes('console')) {
+    logger.info(`Logging to console only'`);
 }
 
 module.exports = logger;

lib/util/settings.js

@@ -1,36 +1,66 @@
 const data = require('./data');
-const file = data.joinPath('configuration.yaml');
+const utils = require('./utils');
+const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
 const objectAssignDeep = require(`object-assign-deep`);
 const path = require('path');
-const fs = require('./fs');
-const onChangeHandlers = [];
+const yaml = require('./yaml');
+const Ajv = require('ajv');
+const ajv = new Ajv({allErrors: true});
 
 const defaults = {
+    whitelist: [],
     ban: [],
     permit_join: false,
     mqtt: {
         include_device_information: false,
     },
-    groups: {},
+    serial: {
+        disable_led: false,
+    },
     device_options: {},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: '#fff8ce',
+                    coordinator: '#e04e5d',
+                    router: '#4ea3e0',
+                },
+                font: {
+                    coordinator: '#ffffff',
+                    router: '#ffffff',
+                    enddevice: '#000000',
+                },
+                line: {
+                    active: '#009900',
+                    inactive: '#994444',
+                },
+            },
+        },
+    },
     experimental: {
-        livolo: false,
-        // json or attribute
+        // json or attribute or attribute_and_json
         output: 'json',
     },
     advanced: {
+        legacy_api: true,
+        log_rotation: true,
+        log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
-        log_level: process.env.DEBUG ? 'debug' : 'info',
+        log_file: 'log.txt',
+        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
         soft_reset_timeout: 0,
         pan_id: 0x1a62,
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         baudrate: 115200,
         rtscts: true,
+        adapter_concurrent: null,
 
         // Availability timeout in seconds, disabled by default.
         availability_timeout: 0,
         availability_blacklist: [],
+        availability_whitelist: [],
 
         /**
          * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
@@ -71,255 +101,581 @@ const defaults = {
          * Home Assistant discovery topic
          */
         homeassistant_discovery_topic: 'homeassistant',
+
+        /**
+         * Home Assistant status topic
+         */
+        homeassistant_status_topic: 'hass/status',
+
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
+        /**
+         * Configurable timestampFormat
+         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
+         */
+        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
 };
 
+const schema = {
+    type: 'object',
+    properties: {
+        device_options: {type: 'object'},
+        homeassistant: {type: 'boolean'},
+        permit_join: {type: 'boolean'},
+        mqtt: {
+            type: 'object',
+            properties: {
+                base_topic: {type: 'string'},
+                server: {type: 'string'},
+                keepalive: {type: 'number'},
+                ca: {type: 'string'},
+                key: {type: 'string'},
+                cert: {type: 'string'},
+                user: {type: 'string'},
+                password: {type: 'string'},
+                client_id: {type: 'string'},
+                reject_unauthorized: {type: 'boolean'},
+                include_device_information: {type: 'boolean'},
+                version: {type: 'number'},
+            },
+            required: ['base_topic', 'server'],
+        },
+        serial: {
+            type: 'object',
+            properties: {
+                port: {type: ['string', 'null']},
+                disable_led: {type: 'boolean'},
+                adapter: {type: 'string', enum: ['deconz', 'zstack']},
+            },
+        },
+        ban: {type: 'array', items: {type: 'string'}},
+        whitelist: {type: 'array', items: {type: 'string'}},
+        experimental: {
+            type: 'object',
+            properties: {
+                transmit_power: {type: 'number'},
+            },
+        },
+        advanced: {
+            type: 'object',
+            properties: {
+                legacy_api: {type: 'boolean'},
+                pan_id: {type: 'number'},
+                ext_pan_id: {type: 'array', items: {type: 'number'}},
+                channel: {type: 'number', minimum: 11, maximum: 26},
+                cache_state: {type: 'boolean'},
+                log_rotation: {type: 'boolean'},
+                log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
+                log_output: {type: 'array', items: {type: 'string'}},
+                log_directory: {type: 'string'},
+                log_file: {type: 'string'},
+                baudrate: {type: 'number'},
+                rtscts: {type: 'boolean'},
+                soft_reset_timeout: {type: 'number', minimum: 0},
+                network_key: {type: 'array', items: {type: 'number'}},
+                last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
+                elapsed: {type: 'boolean'},
+                availability_timeout: {type: 'number', minimum: 0},
+                availability_blacklist: {type: 'array', items: {type: 'string'}},
+                availability_whitelist: {type: 'array', items: {type: 'string'}},
+                report: {type: 'boolean'},
+                homeassistant_discovery_topic: {type: 'string'},
+                homeassistant_status_topic: {type: 'string'},
+                timestamp_format: {type: 'string'},
+                adapter_concurrent: {type: 'number'},
+            },
+        },
+        map_options: {
+            type: 'object',
+            properties: {
+                graphviz: {
+                    type: 'object',
+                    properties: {
+                        colors: {
+                            type: 'object',
+                            properties: {
+                                fill: {
+                                    type: 'object',
+                                    properties: {
+                                        enddevice: {type: 'string'},
+                                        coordinator: {type: 'string'},
+                                        router: {type: 'string'},
+                                    },
+                                },
+                                font: {
+                                    type: 'object',
+                                    properties: {
+                                        enddevice: {type: 'string'},
+                                        coordinator: {type: 'string'},
+                                        router: {type: 'string'},
+                                    },
+                                },
+                                line: {
+                                    type: 'object',
+                                    properties: {
+                                        active: {type: 'string'},
+                                        inactive: {type: 'string'},
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        },
+        devices: {
+            type: 'object',
+            propertyNames: {
+                pattern: '^0x[\\d\\w]{16}$',
+            },
+            patternProperties: {
+                '^.*$': {
+                    type: 'object',
+                    properties: {
+                        friendly_name: {type: 'string'},
+                        retain: {type: 'boolean'},
+                        retention: {type: 'number'},
+                        qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
+                    },
+                    required: ['friendly_name'],
+                },
+            },
+        },
+        groups: {
+            type: 'object',
+            propertyNames: {
+                pattern: '^[\\w].*$',
+            },
+            patternProperties: {
+                '^.*$': {
+                    type: 'object',
+                    properties: {
+                        friendly_name: {type: 'string'},
+                        retain: {type: 'boolean'},
+                        devices: {type: 'array', items: {type: 'string'}},
+                        optimistic: {type: 'boolean'},
+                        qos: {type: 'number'},
+                        filtered_attributes: {type: 'array', items: {type: 'string'}},
+                    },
+                    required: ['friendly_name'],
+                },
+            },
+        },
+    },
+    required: ['homeassistant', 'permit_join', 'mqtt'],
+};
+
+
 let _settings;
-
-function getSettings() {
-    if (!_settings) {
-        _settings = read();
-    }
-
-    if (_settings.hasOwnProperty('advanced') && !_settings.advanced) {
-        delete _settings.advanced;
-    }
-
-    return _settings;
-}
-
-function writeRead() {
-    write();
-    _settings = read();
-    onChangeHandlers.forEach((handler) => handler());
-}
+let _settingsWithDefaults;
 
 function write() {
-    const settings = getSettings();
+    const settings = get();
     const toWrite = objectAssignDeep.noMutate(settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
-    const actual = fs.readYaml(file);
+    const actual = yaml.read(file);
+    if (actual.mqtt && actual.mqtt.password && actual.mqtt.user) {
+        toWrite.mqtt.user = actual.mqtt.user;
+        toWrite.mqtt.password = actual.mqtt.password;
+    }
+
+    if (actual.advanced && actual.advanced.network_key) {
+        toWrite.advanced.network_key = actual.advanced.network_key;
+    }
+
     if (typeof actual.devices === 'string') {
-        fs.writeYaml(data.joinPath(actual.devices), settings.devices);
+        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
         toWrite.devices = actual.devices;
     }
 
     if (typeof actual.groups === 'string') {
-        fs.writeYaml(data.joinPath(actual.groups), settings.groups);
+        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
         toWrite.groups = actual.groups;
     }
 
-    fs.writeYaml(file, toWrite);
+    yaml.writeIfChanged(file, toWrite);
+
+    _settings = read();
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+}
+
+function validate() {
+    const validate = ajv.compile(schema);
+    const valid = validate(_settings);
+    const postfixes = utils.getEndpointNames();
+
+    // Verify that all friendly names are unique
+    const names = [];
+    const check = (name) => {
+        if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
+        if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
+        if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+        if (name.includes('#') || name.includes('+')) {
+            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+        }
+        names.push(name);
+    };
+
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                throw new Error('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+
+    return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
 }
 
 function read() {
-    const s = fs.readYaml(file);
+    const s = yaml.read(file);
+
+    // Read !secret MQTT username and password if set
+    const interpetValue = (value) => {
+        const re = /!(.*) (.*)/g;
+        const match = re.exec(value);
+        if (match) {
+            const file = data.joinPath(`${match[1]}.yaml`);
+            const key = match[2];
+            return yaml.read(file)[key];
+        } else {
+            return value;
+        }
+    };
+
+    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+        s.mqtt.user = interpetValue(s.mqtt.user);
+        s.mqtt.password = interpetValue(s.mqtt.password);
+    }
+
+    if (s.advanced && s.advanced.network_key) {
+        s.advanced.network_key = interpetValue(s.advanced.network_key);
+    }
 
     // Read devices/groups configuration from separate file.
     if (typeof s.devices === 'string') {
         const file = data.joinPath(s.devices);
-        s.devices = fs.readYamlIfExists(file);
+        s.devices = yaml.readIfExists(file) || {};
     }
 
     if (typeof s.groups === 'string') {
         const file = data.joinPath(s.groups);
-        s.groups = fs.readYamlIfExists(file);
+        s.groups = yaml.readIfExists(file) || {};
     }
 
     return s;
 }
 
+function get() {
+    if (!_settings) {
+        _settings = read();
+    }
+
+    return _settings;
+}
+
+function getWithDefaults() {
+    if (!_settingsWithDefaults) {
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    }
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    return _settingsWithDefaults;
+}
+
 function set(path, value) {
-    let obj = getSettings();
+    let settings = get();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
         if (i === path.length - 1) {
-            obj[key] = value;
+            settings[key] = value;
         } else {
-            if (!obj[key]) {
-                obj[key] = {};
+            if (!settings[key]) {
+                settings[key] = {};
             }
 
-            obj = obj[key];
+            settings = settings[key];
         }
     }
 
-    writeRead();
+    write();
 }
 
-const getDevices = () => getSettings().devices || [];
+function getGroup(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.groups[IDorName];
+    if (byID) {
+        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+    }
 
-const getDevice = (ieeeAddr) => getDevices()[ieeeAddr];
+    for (const [ID, group] of Object.entries(settings.groups)) {
+        if (group.friendly_name === IDorName) {
+            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        }
+    }
 
-const getGroups = () => getSettings().groups || [];
+    return null;
+}
 
-const getGroup = (ID) => getGroups()[ID];
+function getGroups() {
+    const settings = getWithDefaults();
+    return Object.entries(settings.groups).map(([ID, group]) => {
+        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+    });
+}
 
+function getGroupThrowIfNotExists(IDorName) {
+    const group = getGroup(IDorName);
+    if (!group) {
+        throw new Error(`Group '${IDorName}' does not exist`);
+    }
+
+    return group;
+}
+
+function getDevice(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.devices[IDorName];
+    if (byID) {
+        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, device] of Object.entries(settings.devices)) {
+        if (device.friendly_name === IDorName) {
+            return {...device, ID, friendlyName: device.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getDeviceThrowIfNotExists(IDorName) {
+    const device = getDevice(IDorName);
+    if (!device) {
+        throw new Error(`Device '${IDorName}' does not exist`);
+    }
+
+    return device;
+}
+
+function getEntity(IDorName) {
+    const device = getDevice(IDorName);
+    if (device) {
+        return {...device, type: 'device'};
+    }
+
+    const group = getGroup(IDorName);
+    if (group) {
+        return {...group, type: 'group'};
+    }
+
+    return null;
+}
+
+function addDevice(ID) {
+    if (getDevice(ID)) {
+        throw new Error(`Device '${ID}' already exists`);
+    }
+
+    const settings = get();
 
-function addDevice(ieeeAddr) {
-    const settings = getSettings();
     if (!settings.devices) {
         settings.devices = {};
     }
 
-    settings.devices[ieeeAddr] = {friendly_name: ieeeAddr, retain: false};
-    writeRead();
+    settings.devices[ID] = {friendly_name: ID};
+    write();
+    return getDevice(ID);
 }
 
-function banDevice(ieeeAddr) {
-    const settings = getSettings();
+function whitelistDevice(ID) {
+    const settings = get();
+    if (!settings.whitelist) {
+        settings.whitelist = [];
+    }
+
+    if (settings.whitelist.includes(ID)) {
+        throw new Error(`Device '${ID}' already whitelisted`);
+    }
+
+    settings.whitelist.push(ID);
+    write();
+}
+
+function banDevice(ID) {
+    const settings = get();
     if (!settings.ban) {
         settings.ban = [];
     }
 
-    settings.ban.push(ieeeAddr);
-
-    writeRead();
+    settings.ban.push(ID);
+    write();
 }
 
-function removeDevice(ieeeAddr) {
-    const settings = getSettings();
-    if (!settings.devices || !settings.devices[ieeeAddr]) return;
-
-    delete settings.devices[ieeeAddr];
-    writeRead();
+function removeDevice(IDorName) {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = get();
+    delete settings.devices[device.ID];
+    write();
 }
 
-function addGroup(groupName) {
-    const settings = getSettings();
+function addGroup(name, ID=null) {
+    if (getGroup(name) || getDevice(name)) {
+        throw new Error(`friendly_name '${name}' is already in use`);
+    }
+
+    const settings = get();
     if (!settings.groups) {
         settings.groups = {};
     }
 
-    let ID = '1';
-    while (settings.groups.hasOwnProperty(ID)) {
-        ID = (Number.parseInt(ID) + 1).toString();
-    }
-
-    settings.groups[ID] = {friendly_name: groupName};
-    writeRead();
-
-    return true;
-}
-
-function removeGroup(name) {
-    const settings = getSettings();
-    if (!settings.groups) return;
-
-    const ID = Object.keys(settings.groups).find((key) => {
-        return settings.groups[key].friendly_name === name;
-    });
-
-    if (ID) {
-        delete settings.groups[ID];
-        writeRead();
-        return true;
-    } else {
-        return false;
-    }
-}
-
-function getIeeeAddrByFriendlyName(friendlyName) {
-    const entry = Object.entries(getDevices()).find(([ieeeAddr, device]) =>
-        device.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function getGroupIDByFriendlyName(friendlyName) {
-    const entry = Object.entries(getGroups()).find(([ID, group]) =>
-        group.friendly_name === friendlyName
-    );
-    return entry && entry[0];
-}
-
-function changeDeviceOptions(ieeeAddr, newOptions) {
-    const settings = getSettings();
-    const currentOptions = settings.devices[ieeeAddr];
-
-    if (!currentOptions) {
-        return;
-    }
-
-    Object.keys(currentOptions).forEach((key) => {
-        if (newOptions[key]) {
-            currentOptions[key] = newOptions[key];
+    if (ID == null) {
+        // look for free ID
+        ID = '1';
+        while (settings.groups.hasOwnProperty(ID)) {
+            ID = (Number.parseInt(ID) + 1).toString();
         }
-    });
-
-    writeRead();
-}
-
-function changeFriendlyName(old, new_) {
-    const settings = getSettings();
-    const ieeeAddr = getIeeeAddrByFriendlyName(old);
-
-    if (!ieeeAddr) {
-        return false;
-    }
-
-    settings.devices[ieeeAddr].friendly_name = new_;
-    writeRead();
-    return true;
-}
-
-// An entity can be either a group or a device.
-function resolveEntity(ID) {
-    let type = null;
-    let friendlyName = null;
-
-    if (module.exports.getIeeeAddrByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a device.
-        friendlyName = ID;
-        ID = module.exports.getIeeeAddrByFriendlyName(ID);
-        type = 'device';
-    } else if (module.exports.getGroupIDByFriendlyName(ID)) {
-        // Check if the ID is a friendly_name of a group.
-        friendlyName = ID;
-        ID = Number(module.exports.getGroupIDByFriendlyName(ID));
-        type = 'group';
-    } else if (module.exports.getGroup(ID)) {
-        friendlyName = module.exports.getGroup(ID).friendly_name;
-        ID = Number(ID);
-        type = 'group';
     } else {
-        // By default it is a device with ID as ID.
-        type = 'device';
-        const device = module.exports.getDevice(ID);
-        friendlyName = device ? device.friendly_name : ID;
+        // ensure provided ID is not in use
+        ID = ID.toString();
+        if (settings.groups.hasOwnProperty(ID)) {
+            throw new Error(`group id '${ID}' is already in use`);
+        }
     }
 
-    return {ID, type, friendlyName};
+    settings.groups[ID] = {friendly_name: name};
+    write();
+
+    return getGroup(ID);
+}
+
+function groupHasDevice(group, keys) {
+    for (const device of group.devices) {
+        const index = keys.indexOf(device);
+        if (index != -1) {
+            return keys[index];
+        }
+    }
+
+    return false;
+}
+
+function addDeviceToGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
+
+    if (!groupHasDevice(group, keys)) {
+        group.devices.push(keys[0]);
+        write();
+    }
+}
+
+function removeDeviceFromGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
+
+    const key = groupHasDevice(group, keys);
+    if (key) {
+        group.devices = group.devices.filter((d) => d != key);
+        write();
+    }
+}
+
+function removeGroup(groupIDorName) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    delete settings.groups[groupID];
+    write();
+}
+
+function changeDeviceOptions(IDorName, newOptions) {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = get();
+    objectAssignDeep(settings.devices[device.ID], newOptions);
+    write();
+}
+
+function changeFriendlyName(IDorName, newName) {
+    if (getGroup(newName) || getDevice(newName)) {
+        throw new Error(`friendly_name '${newName}' is already in use`);
+    }
+
+    const settings = get();
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
 }
 
 module.exports = {
-    get: () => objectAssignDeep.noMutate(defaults, getSettings()),
-    write: () => write(),
-    set: (path, value) => set(path, value),
-
+    validate,
+    get: getWithDefaults,
+    set,
     getDevice,
     getGroup,
     getGroups,
-    getDevices,
-    banDevice: (ieeeAddr) => banDevice(ieeeAddr),
-    addDevice: (ieeeAddr) => addDevice(ieeeAddr),
-    removeDevice: (ieeeAddr) => removeDevice(ieeeAddr),
-    addGroup: (name) => addGroup(name),
-    removeGroup: (name) => removeGroup(name),
+    getEntity,
+    whitelistDevice,
+    banDevice,
+    addDevice,
+    removeDevice,
+    addGroup,
+    removeGroup,
+    addDeviceToGroup,
+    removeDeviceFromGroup,
+    changeDeviceOptions,
+    changeFriendlyName,
 
-    getIeeeAddrByFriendlyName: (friendlyName) => getIeeeAddrByFriendlyName(friendlyName),
-    getGroupIDByFriendlyName: (friendlyName) => getGroupIDByFriendlyName(friendlyName),
-    changeFriendlyName: (old, new_) => changeFriendlyName(old, new_),
-    changeDeviceOptions: (ieeeAddr, options) => changeDeviceOptions(ieeeAddr, options),
-    resolveEntity,
-
-    addOnChangeHandler: (handler) => onChangeHandlers.push(handler),
-
-    // For test
-    _getDefaults: () => {
-        return objectAssignDeep.noMutate(defaults);
+    // For tests only
+    _write: write,
+    _reRead: () => {
+        _settings = read();
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
     },
-    _clear: () => _settings = undefined,
+    _getDefaults: () => defaults,
 };

lib/util/utils.js

@@ -1,5 +1,5 @@
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const logger = require('../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const equals = require('fast-deep-equal');
 
 // Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
 const xiaomiManufacturerID = [4151, 4447];
@@ -27,78 +27,112 @@ function toLocalISOString(dDate) {
         ':' + pad(tzOffset % 60);
 }
 
-const postfixes = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right',
-    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom',
+const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
 ];
 
-function getEndpointByEntityID(zigbee, entityID, epName) {
-    const device = zigbee.getDevice(entityID);
-    if (!device) {
-        logger.error(`Failed to find device with entity ID '${entityID}'`);
-        return;
-    }
-
-    const mappedDevice = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-    if (!mappedDevice) {
-        logger.error(`Device with model ID ${device.modelId} is not supported`);
-        return;
-    }
-
-    let epID = null;
-    if (epName) {
-        if (!mappedDevice.ep) {
-            logger.error(`Device ${mappedDevice.model} doesn't define eps`);
-            return;
-        }
-
-        epID = mappedDevice.ep(device)[epName];
-        if (!epID) {
-            logger.error(`Device ${mappedDevice.model} doesn't have ep named '${epName}'`);
-            return;
-        }
-    }
-
-    const endpoint = zigbee.getEndpoint(entityID, epID);
-    if (!endpoint) {
-        logger.error(`Failed to retrieve for entity ID ${entityID} and endpoint ID ${epID}`);
-    }
-
-    return endpoint;
+function flatten(arr) {
+    return arr.reduce((flat, toFlatten) => {
+        return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
+    }, []);
 }
 
-function getZigbee2mqttVersion(callback) {
-    const git = require('git-last-commit');
-    const packageJSON = require('../../package.json');
-    const version = packageJSON.version;
+const forceEndDevice = flatten(
+    ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
+        .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
+        .map((mappedModel) => mappedModel.zigbeeModel));
 
-    git.getLastCommit((err, commit) => {
-        let commitHash = null;
+async function getZigbee2mqttVersion() {
+    return new Promise((resolve, reject) => {
+        const git = require('git-last-commit');
+        const packageJSON = require('../../package.json');
+        const version = packageJSON.version;
 
-        if (err) {
-            try {
-                commitHash = require('../.hash.json').hash;
-            } catch (error) {
-                commitHash = 'unknown';
+        git.getLastCommit((err, commit) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = require('../../.hash.json').hash;
+                } catch (error) {
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
             }
-        } else {
-            commitHash = commit.shortHash;
-        }
 
-        callback({commitHash, version});
+            resolve({commitHash, version});
+        });
     });
 }
 
+function formatDate(date, type) {
+    let result;
+
+    switch (type) {
+    case 'ISO_8601':
+        result = new Date(date).toISOString();
+        break;
+    case 'ISO_8601_local':
+        result = toLocalISOString(new Date(date));
+        break;
+    case 'epoch':
+        result = date;
+        break;
+    default:
+        throw new Error(`Unsupported type '${type}'`);
+    }
+
+    return result;
+}
+
+function objectHasProperties(object, properties) {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function equalsPartial(object, expected) {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function getObjectsProperty(objects, key, defaultValue) {
+    for (const object of objects) {
+        if (object.hasOwnProperty(key)) {
+            return object[key];
+        }
+    }
+
+    return defaultValue;
+}
+
 module.exports = {
     millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
     secondsToMilliseconds: (seconds) => seconds * 1000,
-    isXiaomiDevice: (device) => device.modelId !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufId),
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufId),
-    isRouter: (device) => device.type === 'Router',
-    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
-    isNumeric: (string) => /^\d+$/.test(string),
-    toLocalISOString: (dDate) => toLocalISOString(dDate),
-    getPostfixes: () => postfixes,
-    getEndpointByEntityID,
     getZigbee2mqttVersion,
+    objectHasProperties,
+    getObjectsProperty,
+    getEndpointNames: () => endpointNames,
+    isXiaomiDevice: (device) => {
+        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+    },
+    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
+    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelID),
+    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
+    formatDate: (date, type) => formatDate(date, type),
+    equalsPartial,
 };

lib/util/yaml.js

@@ -0,0 +1,36 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const equals = require('fast-deep-equal');
+
+function read(file) {
+    try {
+        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.message =
+                `\n\n\n` +
+                `\t====================================================================\n` +
+                `\tYour YAML file '${file}' is invalid\n` +
+                `\tUse e.g. https://jsonformatter.org/yaml-validator to find and fix the issue.\n` +
+                `\t====================================================================\n` +
+                `\n\n` +
+                error.message;
+        }
+
+
+        throw error;
+    }
+}
+
+function readIfExists(file) {
+    return fs.existsSync(file) ? read(file) : null;
+}
+
+function writeIfChanged(file, content) {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.safeDump(content));
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged};

lib/util/zigbeeQueue.js

@@ -1,84 +0,0 @@
-const maxSimultaneouslyRunning = 5;
-const delay = 250;
-const error17Retries = 3;
-
-class ZigbeeQueue {
-    constructor() {
-        this.queue = [];
-        this.active = [];
-        this.timer = null;
-        this.running = false;
-    }
-
-    start() {
-        this.running = true;
-        this.resetTimer();
-    }
-
-    push(entityID, func) {
-        this.queue.push({entityID, func, attempts: 0});
-    }
-
-    stopTimer() {
-        if (this.timer) {
-            clearInterval(this.timer);
-            this.timer = null;
-        }
-    }
-
-    resetTimer() {
-        this.stopTimer();
-
-        this.timer = setInterval(() => {
-            this.executeNext();
-        }, delay);
-    }
-
-    stop() {
-        this.stopTimer();
-        this.running = false;
-    }
-
-    handleJobComplete(job, error) {
-        if (error && error.message === 'rsp error: 17' && job.attempts < error17Retries) {
-            // Error 17 means that the buffer of the ZNP was full,
-            // retry this for a maximum of 3 times.
-            job.attempts++;
-            this.active.splice(this.active.indexOf(job), 1);
-        } else {
-            this.active.splice(this.active.indexOf(job), 1);
-            this.queue.splice(this.queue.indexOf(job), 1);
-        }
-    }
-
-    executeNext() {
-        if (!this.running) {
-            return;
-        }
-
-        const next = this.getNext();
-
-        if (next) {
-            this.active.push(next);
-            next.func((error) => this.handleJobComplete(next, error));
-        }
-    }
-
-    getNext() {
-        if (this.active.length > (maxSimultaneouslyRunning - 1)) {
-            return null;
-        }
-
-        for (let i = 0; i < this.queue.length; i++) {
-            const job = this.queue[i];
-            const activeDeviceJob = this.active.find((j) => j.entityID === job.entityID);
-            if (!activeDeviceJob) {
-                return job;
-            }
-        }
-
-        return null;
-    }
-}
-
-module.exports = ZigbeeQueue;

lib/zapp/cie.js

@@ -1,57 +0,0 @@
-/**
- * This is a fake cie app which makes it possible for some iAS devices to join the network.
- * Based on: https://github.com/zigbeer/zapp-cie
- */
-
-const Ziee = require('ziee');
-const Zive = require('zive');
-const cieClusters = new Ziee();
-
-cieClusters.init('genBasic', 'dir', {value: 1}); // Server Side(Input)
-cieClusters.init('ssIasAce', 'dir', {value: 1}); // Server Side(Input)
-cieClusters.init('ssIasZone', 'dir', {value: 2}); // Client Side(Output)
-cieClusters.init('ssIasWd', 'dir', {value: 2}); // Client Side(Output)
-cieClusters.init('genIdentify', 'dir', {value: 3}); // Server and Client Side(Input/Output)
-
-// Init Attributes Access Control
-cieClusters.init('genBasic', 'acls', {
-    zclVersion: 'R',
-    hwVersion: 'R',
-    manufacturerName: 'R',
-    modelId: 'R',
-    dateCode: 'R',
-    powerSource: 'R',
-    locationDesc: 'RW',
-    physicalEnv: 'RW',
-    deviceEnabled: 'RW',
-});
-
-cieClusters.init('genIdentify', 'acls', {
-    identifyTime: 'RW',
-});
-
-// Init Attributes Value
-cieClusters.init('genBasic', 'attrs', {
-    zclVersion: 1,
-    hwVersion: 1,
-    manufacturerName: 'sivann inc.',
-    modelId: 'hiver0001',
-    dateCode: '20170407',
-    powerSource: 1,
-    locationDesc: '    ',
-    physicalEnv: 0,
-    deviceEnabled: 1,
-});
-
-cieClusters.init('genIdentify', 'attrs', {
-    identifyTime: 0,
-});
-
-// Init Command Response Handler
-cieClusters.init('ssIasZone', 'cmdRsps', {
-    enrollReq: (zapp, argObj, cb) => {},
-    statusChangeNotification: (zapp, argObj, cb) => {},
-});
-
-const cieApp = new Zive({profId: 0x0104, devId: 0x0400, discCmds: []}, cieClusters);
-module.exports = cieApp;

lib/zigbee.js

@@ -1,444 +1,264 @@
-const ZShepherd = require('zigbee-shepherd');
+const ZigbeeHerdsman = require('zigbee-herdsman');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const data = require('./util/data');
+const assert = require('assert');
 const utils = require('./util/utils');
-const ZigbeeQueue = require('./util/zigbeeQueue');
-const cieApp = require('./zapp/cie');
+const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
-const zclId = require('zcl-id');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 
-const advancedSettings = settings.get().advanced;
+const endpointNames = utils.getEndpointNames();
+const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-if (advancedSettings.channel < 11 || advancedSettings.channel > 26) {
-    throw new Error(`'${advancedSettings.channel}' is an invalid channel, use a channel between 11 - 26.`);
-}
-
-const shepherdSettings = {
-    net: {
-        panId: advancedSettings.pan_id,
-        extPanId: advancedSettings.ext_pan_id,
-        channelList: [advancedSettings.channel],
-        precfgkey: settings.get().advanced.network_key,
+const herdsmanSettings = {
+    network: {
+        panID: settings.get().advanced.pan_id,
+        extendedPanID: settings.get().advanced.ext_pan_id,
+        channelList: [settings.get().advanced.channel],
+        networkKey: settings.get().advanced.network_key,
     },
-    dbPath: data.joinPath('database.db'),
-    sp: {
-        baudRate: advancedSettings.baudrate,
-        rtscts: advancedSettings.rtscts,
+    databasePath: data.joinPath('database.db'),
+    databaseBackupPath: data.joinPath('database.db.backup'),
+    backupPath: data.joinPath('coordinator_backup.json'),
+    serialPort: {
+        baudRate: settings.get().advanced.baudrate,
+        rtscts: settings.get().advanced.rtscts,
+        path: settings.get().serial.port,
+        adapter: settings.get().serial.adapter,
+    },
+    adapter: {
+        concurrent: settings.get().advanced.adapter_concurrent,
     },
 };
 
-const defaultCfg = {
-    manufSpec: 0,
-    disDefaultRsp: 0,
-};
-
-// Don't print network key.
-const shepherdSettingsLog = objectAssignDeep.noMutate(shepherdSettings);
-shepherdSettingsLog.net.precfgkey = 'HIDDEN';
-logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettingsLog)}'`);
-
-class Zigbee {
+class Zigbee extends events.EventEmitter {
     constructor() {
-        this.onReady = this.onReady.bind(this);
-        this.onMessage = this.onMessage.bind(this);
-        this.onError = this.onError.bind(this);
-        this.messageHandler = null;
-
-        this.queue = new ZigbeeQueue();
+        super();
+        this.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler.bind(this);
     }
 
-    start(messageHandler, callback) {
-        logger.info(`Starting zigbee-shepherd`);
-        this.messageHandler = messageHandler;
-        this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
+    async start() {
+        logger.info(`Starting zigbee-herdsman...`);
+        const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
+        herdsmanSettingsLog.network.networkKey = 'HIDDEN';
+        logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
 
-        this.shepherd.start((error) => {
-            if (error) {
-                logger.info('Error while starting zigbee-shepherd, attempting to fix... (takes 60 seconds)');
-                this.shepherd.controller._znp.close((() => null));
-
-                setTimeout(() => {
-                    logger.info(`Starting zigbee-shepherd`);
-                    this.shepherd.start((error) => {
-                        if (error) {
-                            logger.error('Error while starting zigbee-shepherd!');
-                            logger.error(
-                                'Press the reset button on the stick (the one closest to the USB) and start again'
-                            );
-                            callback(error);
-                        } else {
-                            this._handleStarted();
-                            callback(null);
-                        }
-                    });
-                }, utils.secondsToMilliseconds(60));
-            } else {
-                this._handleStarted();
-                callback(null);
-            }
-        });
-
-        // Register callbacks.
-        this.shepherd.on('ready', this.onReady);
-        this.shepherd.on('ind', this.onMessage);
-        this.shepherd.on('error', this.onError);
-        this._acceptDevIncoming = this._acceptDevIncoming.bind(this);
-        this.shepherd.acceptDevIncoming = this._acceptDevIncoming;
-    }
-
-    _handleStarted() {
-        this.logStartupInfo();
-
-        this.getAllClients().forEach((device) => {
-            if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing..`);
-                this.removeDevice(device.ieeeAddr, false, () => {});
-            }
-        });
-    }
-
-    _acceptDevIncoming(devInfo, callback) {
-        logger.debug(
-            `Accept device incoming with ieeeAddr '${devInfo.ieeeAddr}' permit join is '${this.getPermitJoin()}'`
-        );
-
-        if (settings.get().ban.includes(devInfo.ieeeAddr)) {
-            logger.info(`Banned device tried to connect (${devInfo.ieeeAddr})`);
-            callback(null, false);
-        } else {
-            logger.debug(`Allowing device '${devInfo.ieeeAddr}' to join`);
-            callback(null, true);
+        try {
+            herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
+            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
+            await this.herdsman.start();
+        } catch (error) {
+            logger.error(`Error while starting zigbee-herdsman`);
+            throw error;
         }
-    }
 
-    logStartupInfo() {
-        logger.info('zigbee-shepherd started');
-        logger.info(`Coordinator firmware version: '${this.getFirmwareVersion()}'`);
-        logger.debug(`zigbee-shepherd info: ${JSON.stringify(this.shepherd.info())}`);
-    }
+        this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
+        this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
+        this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
+        this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
+        this.herdsman.on('message', (data) => this.emit('event', 'message', data));
 
-    getFirmwareVersion() {
-        return this.shepherd.info().firmware.revision;
-    }
+        logger.info('zigbee-herdsman started');
+        logger.info(`Coordinator firmware version: '${JSON.stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
 
-    softReset(callback) {
-        this.shepherd.reset('soft', callback);
-    }
-
-    stop(callback) {
-        this.queue.stop();
-
-        this.shepherd.stop((error) => {
-            logger.info('zigbee-shepherd stopped');
-            callback(error);
-        });
-    }
-
-    onReady() {
-        // Mount cieApp
-        this.shepherd.mount(cieApp, (err, epId) => {
-            if (!err) {
-                logger.debug(`Mounted the cieApp (epId ${epId})`);
-            } else {
-                logger.error(`Failed to mount the cieApp`);
+        for (const device of this.getClients()) {
+            // If a whitelist is used, all other device will be removed from the network.
+            if (settings.get().whitelist.length > 0) {
+                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
+                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+                    device.removeFromNetwork();
+                }
+            } else if (settings.get().ban.includes(device.ieeeAddr)) {
+                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+                device.removeFromNetwork();
             }
-        });
+        }
 
         // Check if we have to turn off the led
         if (settings.get().serial.disable_led) {
-            this.shepherd.controller.request('UTIL', 'ledControl', {ledid: 3, mode: 0});
+            await this.herdsman.setLED(false);
         }
 
-        // Wait some time before we start the queue, many calls skip this queue which hangs the stick
-        setTimeout(() => {
-            this.queue.start();
-        }, 2000);
-
-        logger.info('zigbee-shepherd ready');
+        // Check if we have to set a transmit power
+        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().experimental.transmit_power;
+            await this.herdsman.setTransmitPower(transmitPower);
+            logger.info(`Set transmit power to '${transmitPower}'`);
+        }
     }
 
-    onError(message) {
-        // This event may appear if zigbee-shepherd cannot decode bad packets (invalid checksum).
-        logger.error(message);
+    async getCoordinatorVersion() {
+        return this.herdsman.getCoordinatorVersion();
     }
 
-    permitJoin(permit, callback) {
-        if (permit) {
-            logger.info('Zigbee: allowing new devices to join.');
-        } else {
+    async reset(type) {
+        await this.herdsman.reset(type);
+    }
+
+    async stop() {
+        await this.herdsman.stop();
+        logger.info('zigbee-herdsman stopped');
+    }
+
+    async permitJoin(permit) {
+        permit ?
+            logger.info('Zigbee: allowing new devices to join.') :
             logger.info('Zigbee: disabling joining new devices.');
-        }
 
-        this.shepherd.permitJoin(permit ? 255 : 0, (error) => {
-            if (error) {
-                logger.info(error);
-            }
-
-            if (callback) {
-                callback();
-            }
-        });
+        await this.herdsman.permitJoin(permit);
     }
 
-    getPermitJoin() {
-        return this.shepherd.controller._permitJoinTime === 255;
+    async getPermitJoin() {
+        return this.herdsman.getPermitJoin();
     }
 
-    getAllClients() {
-        return this.getDevices().filter((device) => device.type !== 'Coordinator');
-    }
-
-    removeDevice(deviceID, ban, callback) {
-        if (ban) {
-            settings.banDevice(deviceID);
-        }
-
-        this.shepherd.remove(deviceID, {reJoin: true}, (error) => {
-            if (error) {
-                logger.warn(`Failed to remove '${deviceID}', trying force remove...`);
-                this.forceRemove(deviceID, callback);
-            } else {
-                logger.info(`Removed ${deviceID}`);
-                callback(null);
-            }
-        });
-    }
-
-    forceRemove(deviceID, callback) {
-        const device = this.shepherd._findDevByAddr(deviceID);
-
-        if (device) {
-            return this.shepherd._unregisterDev(device, (error) => {
-                logger.info(`Force removed ${deviceID}`);
-                callback(error);
-            });
-        } else {
-            logger.warn(`Could not find ${deviceID} for force removal`);
-            callback(true);
-        }
-    }
-
-    onMessage(message) {
-        if (this.messageHandler) {
-            this.messageHandler(message);
-        }
+    getClients() {
+        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
     }
 
     getDevices() {
-        return this.shepherd.list();
+        return this.herdsman.getDevices();
     }
 
-    getDevice(ieeeAddr) {
-        return this.getDevices().find((d) => d.ieeeAddr === ieeeAddr);
+    getDeviceByIeeeAddr(ieeeAddr) {
+        return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
     }
 
-    getCoordinator() {
-        const device = this.getDevices().find((d) => d.type === 'Coordinator');
-        return this.shepherd.find(device.ieeeAddr, 1);
+    getDevicesByType(type) {
+        return this.herdsman.getDevicesByType(type);
     }
 
-    getGroup(ID) {
-        return this.shepherd.getGroup(ID);
-    }
-
-    networkScan(callback) {
-        logger.info('Starting network scan...');
-        this.shepherd.lqiScan().then((result) => {
-            logger.info('Network scan completed');
-            callback(result);
-        });
-    }
-
-    getEndpoint(ieeeAddr, ep) {
-        // If no ep is given, the first endpoint will be returned
-        // Find device in zigbee-shepherd
-        const device = this.getDevice(ieeeAddr);
-        if (!device || !device.epList || !device.epList.length) {
-            logger.error(`Zigbee cannot determine endpoint for '${ieeeAddr}'`);
-            return null;
-        }
-
-        ep = ep ? ep : device.epList[0];
-        const endpoint = this.shepherd.find(ieeeAddr, ep);
-        return endpoint;
-    }
-
-    publish(entityID, entityType, cid, cmd, cmdType, zclData, cfg=defaultCfg, ep, callback) {
-        let entity = null;
-        if (entityType === 'device') {
-            entity = this.getEndpoint(entityID, ep);
-        } else if (entityType === 'group') {
-            entity = this.getGroup(entityID);
-        }
-
-        if (!entity) {
-            logger.error(
-                `Cannot publish message to ${entityType} because '${entityID}' is not known by zigbee-shepherd`
-            );
-            return;
-        }
-
-        this.queue.push(entityID, (queueCallback) => {
-            logger.info(
-                `Zigbee publish to ${entityType} '${entityID}', ${cid} - ${cmd} - ` +
-                `${JSON.stringify(zclData)} - ${JSON.stringify(cfg)} - ${ep}`
-            );
-
-            const callback_ = (error, rsp) => {
-                if (error) {
-                    logger.error(
-                        `Zigbee publish to ${entityType} '${entityID}', ${cid} - ${cmd} - ${JSON.stringify(zclData)} ` +
-                        `- ${JSON.stringify(cfg)} - ${ep} ` +
-                        `failed with error ${error}`);
-                }
-
-                if (callback) {
-                    callback(error, rsp);
-                }
-
-                queueCallback(error);
-            };
-
-            if (cmdType === 'functional' && entity.functional) {
-                entity.functional(cid, cmd, zclData, cfg, callback_);
-            } else if (cmdType === 'foundation' && entity.foundation) {
-                entity.foundation(cid, cmd, zclData, cfg, callback_);
-            } else {
-                logger.error(`Unknown zigbee publish cmdType ${cmdType}`);
-            }
-        });
-    }
-
-    ping(ieeeAddr, errorLogLevel='error', cb, mechanism='default') {
-        const callback = (error) => {
-            if (error) {
-                logger[errorLogLevel](`Failed to ping ${ieeeAddr}`);
-            } else {
-                logger.debug(`Successfully pinged ${ieeeAddr}`);
-            }
-
-            if (cb) {
-                cb(error);
-            }
-        };
-
-        if (mechanism === 'default') {
-            const device = this.shepherd._findDevByAddr(ieeeAddr);
-            if (device) {
-                logger.debug(`Ping ${ieeeAddr} (default)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    this.shepherd.controller.checkOnline(device, (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        } else if (mechanism === 'basic') {
-            const endpoint = this.getEndpoint(ieeeAddr, null);
-            if (endpoint) {
-                logger.debug(`Ping ${ieeeAddr} (basic)`);
-                this.queue.push(ieeeAddr, (queueCallback) => {
-                    endpoint.foundation('genBasic', 'read', [{attrId: 0}], (error) => {
-                        callback(error);
-                        queueCallback(error);
-                    });
-                });
-            }
-        }
-    }
-
-    bind(ep, cluster, target, callback) {
-        const log = ` ${ep.device.ieeeAddr} - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Binding ${log}`);
-            ep.bind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to bind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully bound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    unbind(ep, cluster, target, callback) {
-        const log = ` ${ep.device.ieeeAddr} - ${cluster}`;
-        target = !target ? this.getCoordinator() : target;
-
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Unbinding ${log}`);
-            ep.unbind(cluster, target, (error) => {
-                if (error) {
-                    logger.error(`Failed to unbind ${log} - (${error})`);
-                } else {
-                    logger.debug(`Successfully unbound ${log}`);
-                }
-
-                callback(error);
-                queueCallback(error);
-            });
-        });
-    }
-
-    /*
-     * Setup reporting.
-     * Attributes is an array of attribute objects.
-     * each attribute object should contain the following properties:
-     *     attr    the attribute name,
-     *     min     the minimal time between reports in seconds,
-     *     max     the maximum time between reports in seconds,
-     *     change  the minimum amount of change before sending a report
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
      */
-    report(ep, cluster, attributes) {
-        const cfgArr = attributes.map((attribute) => {
-            const attrId = zclId.attr(cluster, attribute.attr).value;
-            const dataType = zclId.attrType(cluster, attribute.attr).value;
-            return {
-                direction: 0,
-                attrId,
-                dataType,
-                minRepIntval: attribute.min,
-                maxRepIntval: attribute.max,
-                repChange: attribute.change,
-            };
-        });
+    resolveEntity(key) {
+        assert(
+            typeof key === 'string' || typeof key === 'number' ||
+            key.constructor.name === 'Device', `Wrong type '${typeof key}'`,
+        );
 
-        const log=`for ${ep.device.ieeeAddr} - ${cluster} - ${attributes.length}`;
+        if (typeof key === 'string' || typeof key === 'number') {
+            if (typeof key === 'number') {
+                key = key.toString();
+            }
 
-        const configReport = () => {
-            this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-                ep.foundation(cluster, 'configReport', cfgArr, defaultCfg, (error) => {
-                    if (error) {
-                        logger.error(`Failed to setup reporting ${log} - (${error})`);
-                    } else {
-                        logger.debug(`Successfully setup reporting ${log}`);
-                    }
+            if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
+                const coordinator = this.getDevicesByType('Coordinator')[0];
+                return {
+                    type: 'device',
+                    device: coordinator,
+                    endpoint: coordinator.getEndpoint(1),
+                    settings: {friendlyName: 'Coordinator'},
+                    name: 'Coordinator',
+                };
+            }
 
-                    queueCallback(error);
-                });
-            });
-        };
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
+            }
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
+            }
 
-        this.queue.push(ep.device.ieeeAddr, (queueCallback) => {
-            logger.debug(`Setup reporting ${log}`);
-
-            ep.bind(cluster, this.getCoordinator(), (error) => {
-                if (error) {
-                    logger.error(`Failed to bind for reporting ${log} - (${error})`);
-                } else {
-                    // Only if binding succeeds, setting-up reporting makes sense.
-                    configReport();
+            const entity = settings.getEntity(key);
+            if (!entity) {
+                return null;
+            } else if (entity.type === 'device') {
+                const device = this.getDeviceByIeeeAddr(entity.ID);
+                if (!device) {
+                    return null;
                 }
 
-                queueCallback(error);
-            });
-        });
+                const definition = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
+                let endpoint;
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
+                    } else {
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
+                        endpoint = device.getEndpoint(endpointID);
+                    }
+                } else if (endpoints && endpoints['default']) {
+                    endpoint = device.getEndpoint(endpoints['default']);
+                } else {
+                    endpoint = device.endpoints[0];
+                }
+
+                return {
+                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
+                };
+            } else {
+                let group = this.getGroupByID(entity.ID);
+                if (!group) group = this.createGroup(entity.ID);
+                return {type: 'group', group, settings: entity, name: entity.friendlyName};
+            }
+        } else {
+            const setting = settings.getEntity(key.ieeeAddr);
+            return {
+                type: 'device',
+                device: key,
+                endpoint: key.endpoints[0],
+                settings: setting,
+                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByZigbeeModel(key.modelID),
+            };
+        }
+    }
+
+    getGroupByID(ID) {
+        return this.herdsman.getGroupByID(ID);
+    }
+
+    getGroups() {
+        return this.herdsman.getGroups();
+    }
+
+    createGroup(groupID) {
+        return this.herdsman.createGroup(groupID);
+    }
+
+    acceptJoiningDeviceHandler(ieeeAddr) {
+        // If set whitelist devices, all other device will be rejected to join the network
+        if (settings.get().whitelist.length > 0) {
+            if (settings.get().whitelist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+                return true;
+            } else {
+                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                return false;
+            }
+        } else if (settings.get().ban.length > 0) {
+            if (settings.get().ban.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+                return false;
+            } else {
+                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        return this.herdsman.touchlinkFactoryReset();
     }
 }
 

package.json

@@ -1,14 +1,14 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.3.1",
-  "description": "Zigbee to MQTT bridge using zigbee-shepherd",
+  "version": "1.12.1",
+  "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "10"
+    "node": "^10 || ^12 || ^13"
   },
   "keywords": [
     "xiaomi",
@@ -20,10 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "coverage": "jest --collect-coverage",
-    "eslint": "node_modules/.bin/eslint .",
+    "test-with-coverage": "jest test --coverage",
+    "eslint": "node_modules/.bin/eslint lib/",
     "start": "node index.js",
-    "test": "jest"
+    "test": "jest test",
+    "test-watch": "jest test --watch"
   },
   "author": "Koen Kanters",
   "license": "GPL-3.0",
@@ -32,28 +33,39 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
+    "ajv": "*",
     "debounce": "*",
+    "fast-deep-equal": "*",
     "git-last-commit": "*",
     "js-yaml": "*",
     "mkdir-recursive": "*",
     "moment": "*",
     "mqtt": "*",
     "object-assign-deep": "*",
-    "queue": "*",
     "rimraf": "*",
     "semver": "*",
-    "winston": "2.4.2",
-    "zcl-id": "git+https://github.com/Koenkk/zcl-id.git#4fa75c92424cd070a3bf6e1f4640b1e3ad8f802d",
-    "ziee": "*",
-    "zigbee-shepherd": "git+https://github.com/Koenkk/zigbee-shepherd.git#a8b5828200140adf4de83457508576f6879f7ec1",
-    "zigbee-shepherd-converters": "8.1.1",
-    "zive": "*"
+    "winston": "*",
+    "zigbee-herdsman": "0.12.76",
+    "zigbee-herdsman-converters": "12.0.69"
   },
   "devDependencies": {
-    "@types/jest": "^24.0.11",
     "eslint": "*",
     "eslint-config-google": "*",
-    "eslint-plugin-jest": "^22.3.0",
-    "jest": "^24.3.1"
+    "eslint-plugin-jest": "*",
+    "jest": "*",
+    "tmp": "*"
+  },
+  "jest": {
+    "coverageThreshold": {
+      "global": {
+        "branches": 100,
+        "functions": 100,
+        "lines": 100,
+        "statements": 100
+      }
+    },
+    "collectCoverageFrom": [
+      "lib/**"
+    ]
   }
 }

scripts/generateNewSupportedDevices.js

@@ -0,0 +1,10 @@
+const fs = require('fs');
+const filename = process.argv[2];
+const text = fs.readFileSync(filename, 'utf8');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+
+for (const line of text.split('\n')) {
+    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
+    console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
+}

scripts/install.sh

@@ -26,7 +26,7 @@ chown -R pi:pi /opt/zigbee2mqtt
 
 echo "Running install. This might take a while and can produce som expected errors"
 cd /opt/zigbee2mqtt || exit
-su pi -c "npm install"
+su pi -c "npm ci"
 
 echo "Creating service file zigbee2mqtt.service"
 service_path="/etc/systemd/system/zigbee2mqtt.service"
@@ -70,4 +70,4 @@ fi
 return 0
 }
 
-[[ "$_" == "$0" ]] && zigbee2mqtt-install-package 
+[[ "$_" == "$0" ]] && zigbee2mqtt-install-package

test/bridgeConfig.test.js

@@ -1,140 +0,0 @@
-const BridgeConfig = require('../lib/extension/bridgeConfig');
-const settings = require('../lib/util/settings');
-const fs = require('../lib/util/fs');
-const objectAssignDeep = require('object-assign-deep');
-const data = require('../lib/util/data');
-const configurationFile = data.joinPath('configuration.yaml');
-
-const mqtt = {
-    subscribe: (topic) => {},
-    log: (type, message) => {},
-};
-
-describe('BridgeConfig', () => {
-    let bridgeConfig;
-    const write = (file, json) => fs.writeYaml(file, json);
-    const read = (file) => fs.readYaml(file);
-    const files = new Map();
-
-    beforeAll(() => {
-        jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            throw new Error(`Fake file not found: ${file}`);
-        });
-        jest.spyOn(fs, 'readYamlIfExists').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            return null;
-        });
-        jest.spyOn(fs, 'writeYaml').mockImplementation((file, content) => {
-            files.set(file, objectAssignDeep.noMutate(content));
-        });
-    });
-
-    beforeEach(() => {
-        settings._clear();
-        files.clear();
-        bridgeConfig = new BridgeConfig(null, mqtt, null, null);
-    });
-
-    afterAll(() => {
-        fs.readYaml.mockRestore();
-        fs.readYamlIfExists.mockRestore();
-        fs.writeYaml.mockRestore();
-    });
-
-    it('Setting elapsed false', async () => {
-        write(configurationFile, {advanced: {elapsed: true}});
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/elapsed', 'false');
-        const expected = {
-            advanced: {
-                elapsed: false,
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Add groups when there are none', async () => {
-        write(configurationFile, {});
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/add_group', 'test123');
-        const expected = {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Add groups when there are', async () => {
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/add_group', 'test1234');
-        const expected = {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Remove group', async () => {
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test123',
-                },
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/remove_group', 'test123');
-        const expected = {
-            groups: {
-                '2': {
-                    friendly_name: 'test1234',
-                },
-            },
-        };
-
-        expect(read(configurationFile)).toStrictEqual(expected);
-    });
-
-    it('Get groups', async () => {
-        jest.spyOn(mqtt, 'log').mockImplementation((type, message) => {
-            expect(type).toBe('groups');
-            expect(Object.keys(message)).toHaveLength(2);
-            expect(message['1'].friendly_name).toBe('test1');
-            expect(message['4711'].friendly_name).toBe('test42');
-        });
-
-        write(configurationFile, {
-            groups: {
-                '1': {
-                    friendly_name: 'test1',
-                },
-                '4711': {
-                    friendly_name: 'test42',
-                },
-            },
-        });
-
-        bridgeConfig.onMQTTMessage('zigbee2mqtt/bridge/config/groups', 'whatever');
-    });
-});

test/bridgeLegacy.test.js

@@ -0,0 +1,463 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+
+describe('Bridge legacy', () => {
+    let controller;
+
+    beforeAll(async () => {
+        this.version = await require('../lib/util/utils').getZigbee2mqttVersion();
+        controller = new Controller();
+        await controller.start();
+    })
+
+    beforeEach(() => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+    });
+
+    it('Should publish bridge configuration on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/config',
+          JSON.stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"log_level":'info',"permit_join":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'false');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'wrong');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+    });
+
+    it('Should allow whitelist', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        const bulb = zigbeeHerdsman.devices.bulb;
+        expect(settings.get().whitelist).toStrictEqual([]);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow changing device options', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', "{friendly_name: 'bulb_color'malformed: {retain: true}}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'false');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+    });
+
+    it('Should allow to reset', async () => {
+        zigbeeHerdsman.reset.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledWith('soft');
+        zigbeeHerdsman.reset.mockImplementationOnce(() => {throw new Error('')});
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.reset.mock.calls[1][0]).toBe('soft');
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'disable');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'notvalid');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+    });
+
+    it('Should allow to set log_level', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'error');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'notvalid');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+    });
+
+    it('Should allow to get devices', async () => {
+        const now = Date.now;
+        Date.now = () => 100;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        Date.now = now;
+    });
+
+    it('Should allow to get groups', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/groups', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+    });
+
+    it('Should allow rename devices', async () => {
+        const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(null);
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', "{old: 'bulb_color2'newmalformed: 'bulb_color3'}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+    });
+
+    it('Should allow rename groups', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        await flushPromises();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt rename when no device has been joined', async () => {
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+    });
+
+    it('Should allow to add groups', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', 'new_group');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+    });
+
+    it('Should allow to add groups with json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying only id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
+    });
+
+    it('Should allow to remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt allow add groups without id or friendly_name in json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{}');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith('Failed to add group, missing friendly_name!');
+    });
+
+    it('Shouldnt do anything on unsupported topic', async () => {
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/not_supported', 'to_be_removed');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to force remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromDatabase.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to ban device', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            JSON.stringify({type: 'device_banned', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.get().ban).toStrictEqual(['0x000b57fffec6a5b3']);
+    });
+
+    it('Shouldnt crash when removing non-existing device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'not_existing_123');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Cannot remove, device 'not_existing_123' does not exist`);
+    });
+
+    it('Should handle when remove fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should handle when ban fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to touchlink factory reset (OK)', async () => {
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
+    });
+
+    it('Should allow to touchlink factory reset (FAILS)', async () => {
+        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
+    });
+});

test/controller.test.js

@@ -1,104 +1,532 @@
-const Controller = require('../lib/controller');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
-const mqtt = require('../lib/mqtt');
-const utils = require('./utils');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
 
 describe('Controller', () => {
     let controller;
-    let mqttPublish;
 
     beforeEach(() => {
-        utils.stubLogger(jest);
-        jest.spyOn(settings, 'getDevice').mockReturnValue({friendly_name: 'test'});
-        mqttPublish = jest.spyOn(mqtt.prototype, 'publish').mockReturnValue(undefined);
+        zigbeeHerdsman.returnDevices.splice(0);
         controller = new Controller();
-        controller.zigbee = {
-            getDevice: () => {
-                return {
-                    modelId: 'TRADFRI bulb E27 CWS opal 600lm',
-                    manufName: 'IKEA',
-                    hwVersion: '1.1',
-                    swBuildId: '2.0',
-                    dateCode: '2018-01-01',
-                };
-            },
-        };
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
     });
 
-    afterEach(() => {
-        jest.restoreAllMocks();
+    it('Start controller', async () => {
+        await controller.start();
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
+        expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
+        expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
+        expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
     });
 
-    describe('Handling zigbee messages', () => {
-        it('Should handle a zigbee message', () => {
-            const device = {ieeeAddr: '0x12345678', modelId: 'TRADFRI bulb E27 CWS opal 600lm'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'devChange', {onOff: 1}, 1);
-            controller.onZigbeeMessage(message);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(mqttPublish.mock.calls[0][1]).toBe(JSON.stringify({state: 'ON'}));
-        });
+    it('Start controller with specific MQTT settings', async () => {
+        const ca = tmp.fileSync().name;
+        fs.writeFileSync(ca, "ca");
+        const key = tmp.fileSync().name;
+        fs.writeFileSync(key, "key");
+        const cert = tmp.fileSync().name;
+        fs.writeFileSync(cert, "cert");
 
-        it('Should handle a zigbee message when include_device_information is set', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: true,
-                },
-                advanced: {
-                    cache_state: false,
-                },
-                experimental: {
-                    output: 'json',
-                },
-            });
-
-            const payload = {
-                'state': 'ON',
-                'device': {
-                    'ieeeAddr': '0x12345678',
-                    'friendlyName': 'test',
-                    'manufName': 'IKEA',
-                    'modelId': 'TRADFRI bulb E27 CWS opal 600lm',
-                    'hwVersion': '1.1',
-                    'swBuildId': '2.0',
-                    'dateCode': '2018-01-01',
-                },
-            };
-
-            const device = {ieeeAddr: '0x12345678', modelId: 'TRADFRI bulb E27 CWS opal 600lm'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'devChange', {onOff: 1}, 1);
-            controller.onZigbeeMessage(message);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual(payload);
+        const configuration = {
+            base_topic: "zigbee2mqtt",
+            server: "mqtt://localhost",
+            keepalive: 30,
+            ca, cert, key,
+            password: 'pass',
+            user: 'user1',
+            client_id: 'my_client_id',
+            reject_unauthorized: false,
+            version: 5,
         }
-        );
+        settings.set(['mqtt'], configuration)
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            keepalive: 30,
+            ca: Buffer.from([99, 97]),
+            key: Buffer.from([107, 101, 121]),
+            cert: Buffer.from([99, 101, 114, 116]),
+            password: 'pass',
+            username: 'user1',
+            clientId: 'my_client_id',
+            rejectUnauthorized: false,
+            protocolVersion: 5,
 
-        it('Should output to json by default', () => {
-            const payload = {temperature: 1, humidity: 2};
-            controller.publishEntityState('0x12345678', payload);
-            expect(mqttPublish).toHaveBeenCalledTimes(1);
-            expect(JSON.parse(mqttPublish.mock.calls[0][1])).toStrictEqual(payload);
-        });
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
 
-        it('Should output to attribute', () => {
-            jest.spyOn(settings, 'get').mockReturnValue({
-                mqtt: {
-                    include_device_information: false,
-                },
-                advanced: {
-                    cache_state: false,
-                },
-                experimental: {
-                    output: 'attribute',
-                },
-            });
+    it('Start controller should publish cached states', async () => {
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+    });
 
-            const payload = {temperature: 1, humidity: 2};
-            controller.publishEntityState('0x12345678', payload);
-            expect(mqttPublish).toHaveBeenCalledTimes(2);
-            expect(mqttPublish.mock.calls[0][0]).toBe('test/temperature');
-            expect(mqttPublish.mock.calls[0][1]).toBe('1');
-            expect(mqttPublish.mock.calls[1][0]).toBe('test/humidity');
-            expect(mqttPublish.mock.calls[1][1]).toBe('2');
-        });
+    it('Start controller should not publish cached states when cache_state is false', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Log when MQTT client is unavailable', async () => {
+        jest.useFakeTimers();
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Dont publish to mqtt when client is unavailable', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(2);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"state\":\"ON\",\"brightness\":50,\"color_temp\":370,\"linkquality\":99,\"color\":{\"r\":100,\"g\":50,\"b\":10},\"dummy\":{\"1\":\"yes\",\"2\":\"no\"}}");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Load empty state when state file does not exist', async () => {
+        data.removeState();
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
+    });
+
+    it('Should remove non whitelisted devices on startup', async () => {
+        settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should remove banned devices on startup', async () => {
+        settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
+    it('Start controller fails', async () => {
+        zigbeeHerdsman.start.mockImplementationOnce(() => {throw new Error('failed')});
+        await controller.start();
+        expect(mockExit).toHaveBeenCalledTimes(1);
+    });
+
+    it('Start controller with permit join true', async () => {
+        settings.set(['permit_join'], false);
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+    });
+
+    it('Refuse to start when configuration.yaml is invalid', async () => {
+        settings.set(['permit_join'], 'invalid');
+        await controller.start();
+        expect(logger.error).toHaveBeenCalledWith('Refusing to start, configuration.yaml is not valid, found the following errors:');
+        expect(logger.error).toHaveBeenCalledWith('\t - permit_join should be boolean');
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Start controller with disable_led', async () => {
+        settings.set(['serial', 'disable_led'], true);
+        await controller.start();
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledWith(false);
+    });
+
+    it('Start controller with transmit power', async () => {
+        settings.set(['experimental', 'transmit_power'], 14);
+        await controller.start();
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledWith(14);
+    });
+
+    it('Start controller and stop', async () => {
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(0);
+    });
+
+    it('Start controller and stop', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Start controller adapter disconnects', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await zigbeeHerdsman.events.adapterDisconnected();
+        await flushPromises();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
+    it('Handle mqtt message', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('dummytopic', 'dummymessage');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
+    });
+
+    it('On zigbee event message', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1`);
+    });
+
+    it('On zigbee event message with group ID', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
+    });
+
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
+    it('On zigbee event message from unkown device should create it', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
+        expect(settings.getDevice(device.ieeeAddr)).toBeNull();
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(settings.getDevice(device.ieeeAddr)).toStrictEqual({"ID": "0x0017880104e45519", "friendlyName": "0x0017880104e45519", "friendly_name": "0x0017880104e45519"});
+    });
+
+    it('On zigbee deviceJoined', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('acceptJoiningDeviceHandler reject banned device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['ban'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['ban'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['ban'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('Shouldnt crash when two device join events are received', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview started', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'started'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview failed', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'failed'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful not supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.unsupported;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee event device announce', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        settings.set(['devices'], {})
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and NOT settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", 'yes', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+
+    it('Publish entity state attribute_json output filtered', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered (device_options)', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state with device information', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'include_device_information'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
+
+        // Unsupported device should have model "unknown"
+        await controller.publishEntityState('unsupported2', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state without retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], false);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state with retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Should publish entity state with expiring retention', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'version'], 5);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retention'], 37);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+    });
+
+    it('Publish entity state no empty messages', async () => {
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Publish should not cache when set', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"state\":\"ON\"}", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"brightness\":200}", {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish should not do anything for unknown entity', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb-unknown', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should start when state is corrupted', async () => {
+        fs.writeFileSync(path.join(data.mockDir, 'state.json'), 'corrupted');
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
     });
 });

test/data.test.js

@@ -1,5 +1,8 @@
+const logger = require('./stub/logger');
 const data = require('../lib/util/data.js');
 const path = require('path');
+const tmp = require('tmp');
+const fs = require('fs');
 
 describe('Data', () => {
     describe('Get path', () => {
@@ -10,11 +13,12 @@ describe('Data', () => {
         });
 
         it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
-            const expected = path.join('var', 'zigbee2mqtt');
+            const expected = tmp.dirSync().name;
             process.env.ZIGBEE2MQTT_DATA = expected;
             data._reload();
             const actual = data.getPath();
             expect(actual).toBe(expected);
+            expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
             delete process.env.ZIGBEE2MQTT_DATA;
             data._reload();
         });

test/deviceAvailability.test.js

@@ -1,46 +1,355 @@
-const Availability = require('../lib/extension/deviceAvailability');
-const utils = require('./utils');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
 
-describe('Availability', () => {
-    let availability;
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        availability = new Availability(null, null, null, () => {});
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Device availability', () => {
+    let controller;
+
+    function getExtension() {
+        return controller.extensions.find((e) => e.constructor.name === 'DeviceAvailability');
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        jest.useFakeTimers();
+        settings.set(['advanced', 'availability_timeout'], 10)
+        controller = new Controller();
+        mocksClear.forEach((m) => m.mockClear());
+        await controller.start();
+        await flushPromises();
     });
 
-    afterEach(() => {
-        jest.restoreAllMocks();
+    afterEach(async () => {
+        await controller.stop();
+        await flushPromises();
+    })
+
+    it('Should publish availabilty on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
-    describe('Determine pingable devices', () => {
-        it('Router device should be a pingable device', () => {
-            const device = {
-                powerSource: 'Mains (single phase)',
-                type: 'Router',
-            };
+    it('Should publish availabilty offline when ping fails', async () => {
+        MQTT.publish.mockClear();
+        logger.error.mockClear();
+        logger.debug.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledTimes(3);
+        expect(logger.debug).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
 
-            expect(availability.isPingable(device)).toBe(true);
-        });
+    it('Should publish availabilty online and query state on reconnect', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
 
-        it('Battery device should not be a pingable device', () => {
-            const device = {
-                powerSource: 'Battery',
-                type: 'EndDevice',
-            };
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+    });
 
-            expect(availability.isPingable(device)).toBe(false);
-        });
+    it('Should fail gracefully when quering state after reconnect fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('Device timedout')});
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        logger.debug.mockClear();
+        jest.advanceTimersByTime(11 * 2000);
+        await flushPromises();
 
-        it('E11-G13 should be a pingable device', () => {
-            const device = {
-                powerSource: 'Mains (single phase)',
-                type: 'EndDevice',
-                modelId: 'E11-G13',
-                manufId: 4448,
-            };
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to read state of 'bulb_color' after reconnect`);
+    });
 
-            expect(availability.isPingable(device)).toBe(true);
-        });
+    it('Shouldnt ping again when still pinging', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        device.ping.mockImplementationOnce(async () => {await wait(100000000)});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should mark device online when receiving message while offline', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
+        MQTT.publish.mockClear();
+        getExtension().state = {};
+        const payload = {device: zigbeeHerdsman.devices.bulb_color};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout on deviceAnnounce', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+    });
+
+    it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not do anything when message has no device', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
+    });
+
+    it('Should not ping devices blacklisted by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices blacklisted by IEEE address', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should ping forced pingable devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        getExtension().state[device.ieeeAddr] = false;
+        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+    });
+
+    it('Should not read when device has no modelID and reconnects', async () => {
+        const device = zigbeeHerdsman.devices.nomodel;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not read when device has is unsupported', async () => {
+        const device = zigbeeHerdsman.devices.unsupported_router;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should stop pinging device when removed', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        const endpoint = device.getEndpoint(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should publish availability when end device joins', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const payload = {device};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button_double_key/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should mark non-pingable device as non-available when offline for longer than 24 hours', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const defaultLastSeen = device.lastSeen;
+        device.lastSeen = Date.now();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        device.lastSeen = device.lastSeen - (1000 * 60 * 60 * 25);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        // Shouldn't do anything more when device is removed
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+
+        device.lastSeen = defaultLastSeen;
     });
 });

test/deviceBind.test.js

@@ -1,177 +1,203 @@
-const DeviceBind = require('../lib/extension/deviceBind');
-const utils = require('./utils');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
 
-const mqtt = {
-    subscribe: () => {},
-    log: () => {},
-};
+describe('Device bind', () => {
+    let controller;
 
-const devices = {
-    bulb: {
-        getSimpleDesc: () => {
-            return {inClusterList: [6, 8]};
-        },
-    },
-    remote: {
-        getSimpleDesc: () => {
-            return {outClusterList: [5, 6, 8]};
-        },
-    },
-    switch_ep2: {
-        getSimpleDesc: () => {
-            return {outClusterList: [5, 6]};
-        },
-    },
-    switch_ep3: {
-        getSimpleDesc: () => {
-            return {inClusterList: [5, 6]};
-        },
-    },
-};
-
-const zigbee = {
-    bind: jest.fn((ep, cluster, target, callback) => {
-        callback(false, null);
-    }),
-    unbind: jest.fn((ep, cluster, target, callback) => {
-        callback(false, null);
-    }),
-    getEndpoint: (ID, ep) => {
-        if (ID === 'bulb' || ID === '0x002') {
-            return devices.bulb;
-        } else if (ID === 'remote' || ID === '0x001') {
-            return devices.remote;
-        } else if (ep == 2 && ID === 'switch_ep2') {
-            return devices.switch_ep2;
-        } else if (ep == 3 && ID === 'switch_ep3') {
-            return devices.switch_ep3;
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
         }
+    }
 
-        throw new Error(`No mock for ${ID} and ep ${ep}`);
-    },
-    getDevice: (ID) => {
-        const lookup = {
-            'switch_ep2': 'lumi.sensor_86sw2.es1',
-            'switch_ep3': 'DNCKAT_S003',
-            'bulb': 'TRADFRI bulb E27 WS opal 980lm',
-            'remote': 'TRADFRI remote control',
-            '0x002': 'TRADFRI bulb E27 WS opal 980lm',
-            '0x001': 'TRADFRI remote control',
-        };
-
-        if (lookup.hasOwnProperty(ID)) {
-            return {modelId: lookup[ID]};
-        }
-
-        throw new Error(`No mock for ${ID}`);
-    },
-};
-
-describe('DeviceBind', () => {
-    let deviceBind;
-
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        deviceBind = new DeviceBind(zigbee, mqtt, null, null);
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        MQTT.publish.mockClear();
     });
 
-    afterEach(() => {
-        zigbee.bind.mockClear();
-        zigbee.unbind.mockClear();
-        jest.restoreAllMocks();
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
     });
 
-    describe('Bind devices', () => {
-        it('Bind', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/remote', 'bulb');
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind by friendly name', async () => {
-            jest.spyOn(settings, 'getIeeeAddrByFriendlyName').mockImplementation((friendlyName) => {
-                const lookup = {
-                    remote: '0x001',
-                    bulb: '0x002',
-                };
-
-                return lookup[friendlyName];
-            });
-
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/remote', 'bulb');
-
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind non default ep', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/switch_ep2/right', 'bulb');
-            expect(zigbee.bind).toHaveBeenCalledTimes(1);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.switch_ep2,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
-
-        it('Bind non default ep to target with non default ep', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/bind/switch_ep2/right', 'switch_ep3/right');
-            expect(zigbee.bind).toHaveBeenCalledTimes(2);
-            expect(zigbee.bind).toHaveBeenNthCalledWith(1,
-                devices.switch_ep2,
-                5,
-                devices.switch_ep3,
-                expect.any(Function)
-            );
-            expect(zigbee.bind).toHaveBeenNthCalledWith(2,
-                devices.switch_ep2,
-                6,
-                devices.switch_ep3,
-                expect.any(Function)
-            );
-        });
+    it('Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    describe('Unbind devices', () => {
-        it('Unbind', async () => {
-            deviceBind.onMQTTMessage('zigbee2mqtt/bridge/unbind/remote', 'bulb');
-            expect(zigbee.unbind).toHaveBeenCalledTimes(2);
-            expect(zigbee.unbind).toHaveBeenNthCalledWith(1,
-                devices.remote,
-                6,
-                devices.bulb,
-                expect.any(Function)
-            );
-            expect(zigbee.unbind).toHaveBeenNthCalledWith(2,
-                devices.remote,
-                8,
-                devices.bulb,
-                expect.any(Function)
-            );
-        });
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'Coordinator');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'group_1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', '1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch_double/right');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
     });
 });

test/deviceConfigure.test.js

@@ -0,0 +1,179 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Device configure', () => {
+    let controller;
+
+    expectRemoteConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(2);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+
+        const endpoint2 = device.getEndpoint(2);
+        expect(endpoint2.write).toHaveBeenCalledTimes(1);
+        expect(endpoint2.write).toHaveBeenCalledWith("genBasic", {"49": {"type": 25, "value": 11}}, {"disableDefaultResponse": true, "manufacturerCode": 4107});
+        expect(device.meta.configured).toBe(1);
+    }
+
+    expectRemoteNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(0);
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+    });
+
+    it('Should configure on startup', async () => {
+        expectRemoteConfigured();
+    });
+
+    it('Should re-configure when device rejoins', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        await flushPromises();
+        mockClear(device);
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should not configure twice', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure on zigbee message when not configured yet', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should allow to reconfigure manually', async () => {
+        expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Shouldnt manually reconfigure when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
+    });
+
+    it('Should skip reconfigure when device does not require this', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+        await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+    });
+
+    it('Should not configure when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewing = true;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteNotConfigured();
+        device.interviewing = false;
+    });
+
+    it('Should configure when not interviewCompleted', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewCompleted = false;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+        device.interviewCompleted = true;
+    });
+
+    it('Should not configure when already configuring', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(500));
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should configure max 3 times when fails', async () => {
+        controller.extensions.find((e) => e.constructor.name === 'DeviceConfigure').attempts = {};
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+});

test/deviceEvent.test.js

@@ -0,0 +1,75 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const mockOnEvent = jest.fn();
+const mappedLivolo = zigbeeHerdsmanConverters.findByZigbeeModel(zigbeeHerdsman.devices.LIVOLO.modelID);
+mappedLivolo.onEvent = mockOnEvent;
+zigbeeHerdsmanConverters.onEvent = jest.fn();
+
+describe('Device event', () => {
+    let controller;
+    const device = zigbeeHerdsman.devices.LIVOLO;
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        zigbeeHerdsmanConverters.onEvent.mockClear();
+        await flushPromises();
+    });
+
+    it('Should call with start event', async () => {
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('start')
+        expect(call[1]).toStrictEqual({})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call with stop event', async () => {
+        mockOnEvent.mockClear();
+        await controller.stop();
+        await flushPromises();
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('stop')
+        expect(call[1]).toStrictEqual({})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call with zigbee event', async () => {
+        mockOnEvent.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(mockOnEvent).toHaveBeenCalledTimes(1);
+        const call = mockOnEvent.mock.calls[0];
+        expect(call[0]).toBe('deviceAnnounce')
+        expect(call[1]).toStrictEqual({device})
+        expect(call[2]).toBe(device);
+    });
+
+    it('Should call index onEvent with zigbee event', async () => {
+        zigbeeHerdsmanConverters.onEvent.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.onEvent).toHaveBeenCalledTimes(1);
+        const call = zigbeeHerdsmanConverters.onEvent.mock.calls[0];
+        expect(call[0]).toBe('deviceAnnounce')
+        expect(call[1]).toStrictEqual({device})
+        expect(call[2]).toBe(device);
+    });
+});

test/deviceReceive.test.js

@@ -1,268 +1,456 @@
-const DeviceReceive = require('../lib/extension/deviceReceive');
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
-const devices = require('zigbee-shepherd-converters').devices;
-const utils = require('./utils');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
 
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
-// Devices
-const WXKG11LM = devices.find((d) => d.model === 'WXKG11LM');
-const WXKG02LM = devices.find((d) => d.model === 'WXKG02LM');
-const WSDCGQ11LM = devices.find((d) => d.model === 'WSDCGQ11LM');
-const RTCGQ11LM = devices.find((d) => d.model === 'RTCGQ11LM');
-const ZNCZ02LM = devices.find((d) => d.model === 'ZNCZ02LM');
+describe('Device receive', () => {
+    let controller;
 
-const mqtt = {
-    log: () => {},
-};
-
-describe('DeviceReceive', () => {
-    let deviceReceive;
-    let publishEntityState;
-
-    beforeEach(() => {
-        utils.stubLogger(jest);
-        jest.spyOn(settings, 'addDevice').mockReturnValue(undefined);
-        publishEntityState = jest.fn();
-        deviceReceive = new DeviceReceive(null, mqtt, null, publishEntityState);
+    beforeEach(async () => {
+        jest.useRealTimers();
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
     });
 
-    afterEach(() => {
-        jest.restoreAllMocks();
+    it('Should handle a zigbee message', async () => {
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    describe('Handling zigbee messages', () => {
-        it('Should handle a zigbee message', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'single'});
-        });
 
-        it('Should handle a zigbee message which uses ep (left)', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'left'});
-        });
+    it('Should handle a zigbee message and counter it when Home Assistant integration is enabled', async () => {
+        settings.set(['homeassistant'], true);
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/button');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({click: '', linkquality: 10});
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should handle a zigbee message which uses ep (right)', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 2);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({click: 'right'});
-        });
+    it('Should handle a zigbee message which uses ep (left)', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should handle a zigbee message with default precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.85});
-        });
+    it('Should handle a zigbee message which uses ep (right)', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(2), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should debounce messages', async () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({debounce: 0.1});
-            const message1 = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: 8}, 1
-            );
-            const message2 = utils.zigbeeMessage(
-                device, 'msRelativeHumidity', 'attReport', {measuredValue: 1}, 1
-            );
-            const message3 = utils.zigbeeMessage(
-                device, 'msPressureMeasurement', 'attReport', {measuredValue: 2}, 1
-            );
-            deviceReceive.onZigbeeMessage(message1, device, WSDCGQ11LM);
-            deviceReceive.onZigbeeMessage(message2, device, WSDCGQ11LM);
-            deviceReceive.onZigbeeMessage(message3, device, WSDCGQ11LM);
-            await wait(200);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
+    it('Should handle a zigbee message with default precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-            deviceReceive.onZigbeeMessage(message1, device, WSDCGQ11LM);
-            await wait(200);
-            expect(publishEntityState).toHaveBeenCalledTimes(2);
-            expect(publishEntityState.mock.calls[1][1]).toStrictEqual({temperature: 0.08});
-        });
+    it('Should debounce messages', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        const data1 = {measuredValue: 8}
+        const payload1 = {data: data1, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        const data2 = {measuredValue: 1}
+        const payload2 = {data: data2, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        const data3 = {measuredValue: 2}
+        const payload3 = {data: data3, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload3);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-        it('Should handle a zigbee message with 1 precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({temperature_precision: 1});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.8});
-        });
+    it('Should debounce and retain messages when set via device_options', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'debounce'], 0.1);
+        settings.set(['device_options', 'retain'], true);
+        delete settings.get().devices['0x0017880104e45522']['retain'];
+        const data1 = {measuredValue: 8}
+        const payload1 = {data: data1, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload1);
+        const data2 = {measuredValue: 1}
+        const payload2 = {data: data2, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload2);
+        const data3 = {measuredValue: 2}
+        const payload3 = {data: data3, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload3);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
+    });
 
-        it('Should handle a zigbee message with 0 precision', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'getDevice').mockReturnValue({temperature_precision: 0});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -1});
-        });
+    it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
+        settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
+        const tempMsg = {data: {measuredValue: 8}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(tempMsg);
+        const pressureMsg = {data: {measuredValue: 2}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(pressureMsg);
+        const tempMsg2 = {data: {measuredValue: 7}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(tempMsg2);
+        const humidityMsg = {data: {measuredValue: 3}, cluster: 'msRelativeHumidity', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 13};
+        await zigbeeHerdsman.events.message(humidityMsg);
+        await flushPromises();
+        jest.advanceTimersByTime(50);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2, linkquality: 13});
+        jest.runAllTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
+    });
 
-        it('Should handle a zigbee message with 1 precision when set via device_options', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'get').mockReturnValue({
-                device_options: {
-                    temperature_precision: 1,
-                },
-                advanced: {
-                    last_seen: 'disable',
-                },
-            });
-            jest.spyOn(settings, 'getDevice').mockReturnValue({});
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.8});
-        }
-        );
+    it('Should handle a zigbee message with 1 precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-        it('Should handle a zigbee message with 2 precision when overrides device_options', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            jest.spyOn(settings, 'get').mockReturnValue({
-                device_options: {
-                    temperature_precision: 1,
-                },
-                advanced: {
-                    last_seen: 'disable',
-                },
-            });
-            jest.spyOn(settings, 'getDevice').mockReturnValue({
-                temperature_precision: 2,
-            });
-            const message = utils.zigbeeMessage(
-                device, 'msTemperatureMeasurement', 'attReport', {measuredValue: -85}, 1
-            );
-            deviceReceive.onZigbeeMessage(message, device, WSDCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({temperature: -0.85});
-        }
-        );
+    it('Should handle a zigbee message with 0 precision', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 0);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-        it('Should handle a zigbee message with voltage 3010', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 3010}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 100, voltage: 3010};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
+    it('Should handle a zigbee message with 1 precision when set via device_options', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'temperature_precision'], 1);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-        it('Should handle a zigbee message with voltage 2850', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2850}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 35, voltage: 2850};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
+    it('Should handle a zigbee message with 2 precision when overrides device_options', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['device_options', 'temperature_precision'], 1);
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 0);
+        const data = {measuredValue: -85}
+        const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
+    });
 
-        it('Should handle a zigbee message with voltage 2650', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2650}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 14, voltage: 2650};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
+    it('WSDCGQ11LM pressure precision from non ZCL properties', async () => {
+        const device = zigbeeHerdsman.devices.WSDCGQ11LM;
+        settings.set(['devices', device.ieeeAddr, 'temperature_precision'], 1);
 
-        it('Should handle a zigbee message with voltage 2000', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', {'65281': {'1': 2000}}, 1);
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {battery: 0, voltage: 2000};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
+        MQTT.publish.mockClear();
+        let payload = {data: {"65281":{"1":2985,"4":5032,"5":9,"6":[0,1],"10":0,"100":2345,"101":4608,"102":91552}}, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
 
-        it('Should publish 1 message when converted twice', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const payload = {
-                '65281': {'1': 3045, '3': 19, '4': 17320, '5': 35, '6': [0, 3], '10': 51107, '11': 381, '100': 0},
-            };
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', payload, 1);
-            deviceReceive.onZigbeeMessage(message, device, RTCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            const expected = {'battery': 100, 'illuminance': 381, 'voltage': 3045};
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual(expected);
-        });
+        MQTT.publish.mockClear();
+        payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+    });
 
-        it('Should publish no message when converted without result', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const payload = {'9999': {'1': 3045}};
-            const message = utils.zigbeeMessage(device, 'genBasic', 'attReport', payload, 1);
-            deviceReceive.onZigbeeMessage(message, device, RTCGQ11LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(0);
-        });
+    it('Should handle a zigbee message with voltage 3010', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 3010}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should publish last_seen epoch', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'epoch',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('number');
-        });
+    it('Should handle a zigbee message with voltage 2850', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2850}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should publish last_seen ISO_8601', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'ISO_8601',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('string');
-        });
+    it('Should handle a zigbee message with voltage 2650', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2650}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should publish last_seen ISO_8601_local', () => {
-            const device = {ieeeAddr: '0x12345678'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1);
-            jest.spyOn(settings, 'get').mockReturnValue({
-                advanced: {
-                    last_seen: 'ISO_8601_local',
-                },
-            });
-            deviceReceive.onZigbeeMessage(message, device, WXKG02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(typeof publishEntityState.mock.calls[0][1].last_seen).toBe('string');
-        });
+    it('Should handle a zigbee message with voltage 2000', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const data = {'65281': {'1': 2000}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should not handle messages forwarded Xiaomi messages', () => {
-            const device = {ieeeAddr: '0x12345678', manufId: 4151, type: 'Router'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1}, 1, 599);
-            deviceReceive.onZigbeeMessage(message, device, ZNCZ02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(0);
-        });
+    it('Should publish 1 message when converted twice', async () => {
+        const device = zigbeeHerdsman.devices.RTCGQ11LM;
+        const data = {'65281': {'1': 3045, '3': 19, '4': 17320, '5': 35, '6': [0, 3], '10': 51107, '11': 381, '100': 0}}
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
 
-        it('Should handle messages from Xiaomi router devices', () => {
-            const device = {ieeeAddr: '0x12345678', manufId: 4151, type: 'Router'};
-            const message = utils.zigbeeMessage(device, 'genOnOff', 'attReport', {onOff: 1});
-            deviceReceive.onZigbeeMessage(message, device, ZNCZ02LM);
-            expect(publishEntityState).toHaveBeenCalledTimes(1);
-            expect(publishEntityState.mock.calls[0][1]).toStrictEqual({state: 'ON'});
-        });
+    it('Should publish 1 message when converted twice', async () => {
+        const device = zigbeeHerdsman.devices.RTCGQ11LM;
+        const data = {'9999': {'1': 3045}};
+        const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish last_seen epoch', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish last_seen ISO_8601', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'ISO_8601');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('string')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should publish last_seen ISO_8601_local', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM;
+        settings.set(['advanced', 'last_seen'], 'ISO_8601_local');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('string')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should not handle messages forwarded Xiaomi messages', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 599};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should handle messages from Xiaomi router devices', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/power_plug');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'state': 'ON', linkquality: 20});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should not handle messages from coordinator', async () => {
+        const device = zigbeeHerdsman.devices.coordinator;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not handle messages from unsupported devices', async () => {
+        const device = zigbeeHerdsman.devices.unsupported;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+        const device = zigbeeHerdsman.devices.interviewing;
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+    });
+
+    it('Should handle a command', async () => {
+        const device = zigbeeHerdsman.devices.E1743;
+        const data = {};
+        const payload = {data, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', linkquality: 10});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
+    it('Should add elapsed', async () => {
+        settings.set(['advanced', 'elapsed'], true);
+        const device = zigbeeHerdsman.devices.E1743;
+        const payload = {data: {}, cluster: 'genLevelCtrl', device, endpoint: device.getEndpoint(1), type: 'commandStopWithOnOff'};
+        const oldNow = Date.now;
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(new Date(150));
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        Date.now.mockReturnValue(new Date(200));
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop'});
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1]).elapsed).toBe(50);
+        expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
+        Date.now = oldNow;
+    });
+
+    it('Should log when message is from supported device but has no converters', async () => {
+        const device = zigbeeHerdsman.devices.ZNCZ02LM;
+        const data = {inactiveText: 'hello'};
+        const payload = {data, cluster: 'genBinaryOutput', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(logger.debug).toHaveBeenCalledWith('No converter available for \'ZNCZ02LM\' with cluster \'genBinaryOutput\' and type \'attributeReport\' and data \'{"inactiveText":"hello"}\'');
+    });
+
+    it('Should report correct energy and power values for different versions of SP600', async () => {
+        // https://github.com/Koenkk/zigbee-herdsman-converters/issues/915, OLD and NEW use different date code
+        // divisor of OLD is not correct and therefore underreports by factor 10.
+        const data = {instantaneousDemand:496,currentSummDelivered:[0,6648]}
+
+        const SP600_NEW = zigbeeHerdsman.devices.SP600_NEW;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_NEW, endpoint: SP600_NEW.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+
+        MQTT.publish.mockClear();
+        const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
+        await zigbeeHerdsman.events.message({data, cluster: 'seMetering', device: SP600_OLD, endpoint: SP600_OLD.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
     });
 });

test/deviceReport.test.js

@@ -0,0 +1,245 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
+
+describe('Device report', () => {
+    let controller;
+
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        settings.set(['advanced', 'report'], true);
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    it('Should configure reporting on startup', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        expectOnOffBrightnessColorReport(endpoint, true);
+    });
+
+    it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        device.save.mockClear();
+        mockClear(device);
+        delete device.meta.reporting;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+        expect(device.save).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not configure reporting when still configuring', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(1000));
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        jest.runAllTimers();
+        jest.useRealTimers();
+    });
+
+    it('Should not mark as configured when reporting setup fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
+    it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for end devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure reporting when deviceAnnounce message from IKEA device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+    });
+
+    it('Should not configure reporting on device leave', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for the CC2530 router', async () => {
+        const device = zigbeeHerdsman.devices.CC2530_ROUTER;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting again when it already failed once', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote and should', async () => {
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
+        const device = zigbeeHerdsman.devices.ZNLDP12LM;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+});

test/group.test.js

@@ -0,0 +1,460 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+const MQTT = require('./stub/mqtt');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const settings = require('../lib/util/settings');
+
+describe('Groups', () => {
+    let controller;
+
+    beforeEach(() => {
+        data.writeEmptyState();
+        controller = new Controller();
+        Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        MQTT.publish.mockClear();
+    })
+
+    it('Apply group updates add', async () => {
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
+        zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
+    });
+
+    it('Apply group updates remove', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+    });
+
+    it('Move to non existing group', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+    });
+
+    it('Add non standard endpoint to group with name', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
+    });
+
+    it('Add non standard endpoint to group with number', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
+    });
+
+    it('Shouldnt crash on non-existing devices', async () => {
+        logger.error.mockClear();
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+        expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
+    });
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}', {"retain": false, qos: 0}, expect.any(Function));
+
+        // Test if subscribed to topics with slashes
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Remove from group all deprecated', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Log when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
+    });
+
+    it('Log when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Device 'bulb_color_not_existing' does not exist");
+    });
+
+    it('Should publish group state change when a device in it changes state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state, filtered', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt publish group state change when a group is not optimistic', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of another group with shared device when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
+            '3': {friendly_name: 'group_3', retain: false, devices: []}
+        });
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via device', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off if all lights within turn off', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF"}', {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off even when missing current state', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller.state.state = {};
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF"}', {"retain": false, qos: 0}, expect.any(Function));
+    });
+});

test/logger.test.js

@@ -0,0 +1,111 @@
+const tmp = require('tmp');
+const dir = tmp.dirSync();
+let settings;
+const fs = require('fs');
+const path = require('path');
+const data = require('./stub/data');
+let stdOutWriteOriginal;
+const rimraf = require('rimraf');
+
+describe('Logger', () => {
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        jest.resetModules();
+        settings = require('../lib/util/settings');
+        settings.set(['advanced', 'log_directory'], dir.name + '/%TIMESTAMP%');
+        settings._reRead();
+        stdOutWriteOriginal = console._stdout.write;
+        console._stdout.write = () => {};
+    });
+
+    afterEach(async () => {
+        console._stdout.write = stdOutWriteOriginal;
+    });
+
+    it('Create log directory', () => {
+        const logger = require('../lib/util/logger.js');
+        const dirs = fs.readdirSync(dir.name);
+        expect(dirs.length).toBe(1);
+    });
+
+    it('Should cleanup', () => {
+        const logger = require('../lib/util/logger.js');
+
+        for (const d of fs.readdirSync(dir.name)) {
+            rimraf.sync(path.join(dir.name, d));
+        }
+
+        for (let i = 0; i < 21; i++) {
+            fs.mkdirSync(path.join(dir.name, `log_${i}`));
+        }
+
+        expect(fs.readdirSync(dir.name).length).toBe(21);
+        logger.cleanup();
+        expect(fs.readdirSync(dir.name).length).toBe(10);
+    })
+
+    it('Should not cleanup when there is no timestamp set', () => {
+        const logger = require('../lib/util/logger.js');
+        for (let i = 30; i < 40; i++) {
+            fs.mkdirSync(path.join(dir.name, `log_${i}`));
+        }
+
+        settings.set(['advanced', 'log_directory'], dir.name + '/bla');
+        expect(fs.readdirSync(dir.name).length).toBe(20);
+        logger.cleanup();
+        expect(fs.readdirSync(dir.name).length).toBe(20);
+    })
+
+    it('Set and get log level', () => {
+        const logger = require('../lib/util/logger.js');
+        logger.setLevel('debug');
+        expect(logger.getLevel()).toBe('debug');
+    });
+
+    it('Logger should be console and file by default', () => {
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.length).toBe(2);
+        expect(pipes[0].constructor.name).toBe('Console');
+        expect(pipes[0].silent).toBe(false);
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].dirname.startsWith(dir.name)).toBeTruthy();
+    });
+
+    it('Logger can be file only', () => {
+        settings.set(['advanced', 'log_output'], ['file']);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.length).toBe(2);
+        expect(pipes[0].constructor.name).toBe('Console');
+        expect(pipes[0].silent).toBe(true);
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].dirname.startsWith(dir.name)).toBeTruthy();
+    });
+
+    it('Logger can be console only', () => {
+        settings.set(['advanced', 'log_output'], ['console']);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.constructor.name).toBe('Console');
+        expect(pipes.silent).toBe(false);
+    });
+
+    it('Logger can be nothing', () => {
+        settings.set(['advanced', 'log_output'], []);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes.constructor.name).toBe('Console');
+        expect(pipes.silent).toBe(true);
+    });
+
+    it('Should allow to disable log rotation', () => {
+        settings.set(['advanced', 'log_rotation'], false);
+        const logger = require('../lib/util/logger.js');
+        const pipes = logger._readableState.pipes;
+        expect(pipes[1].constructor.name).toBe('File');
+        expect(pipes[1].maxFiles).toBeNull();
+        expect(pipes[1].tailable).toBeFalsy();
+        expect(pipes[1].maxsize).toBeNull();
+    });
+});

test/networkMap.test.js

@@ -0,0 +1,171 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+
+const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
+
+zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(bulb_color.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(WXKG02LM.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(CC2530_ROUTER.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(unsupported_router.ieeeAddr);
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+Date.now = jest.fn()
+Date.now.mockReturnValue(10000);
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Networkmap', () => {
+    let controller;
+
+    beforeAll(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    function mock() {
+        /**
+         * Topology
+         *
+         *             | -> bulb_color -> unsupported_router (offline)
+         * coordinator |      ^     ^
+         *             |      |     | (not valid)
+         *             | -> bulb    |
+         *                    |  -> CC2530_ROUTER -> WXKG02LM
+         *
+         */
+        coordinator.lqi = () => {return {neighbors: [
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 2, depth: 1, linkquality: 120},
+            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92}
+        ]}};
+        coordinator.routingTable = () => {return {table: [
+            {destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress},
+        ]}};
+
+        bulb.lqi = () => {return {neighbors: [
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 1, depth: 2, linkquality: 110},
+            {ieeeAddr: CC2530_ROUTER.ieeeAddr, networkAddress: CC2530_ROUTER.networkAddress, relationship: 1, depth: 2, linkquality: 100}
+        ]}};
+        bulb.routingTable = () => {return {table: []}};
+
+        bulb_color.lqi = () => {return {neighbors: []}}
+        bulb_color.routingTable = () => {return {table: []}};
+
+        CC2530_ROUTER.lqi = () => {return {neighbors: [
+            {ieeeAddr: WXKG02LM.ieeeAddr, networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
+        ]}};
+        CC2530_ROUTER.routingTable = () => {return {table: []}};
+
+        unsupported_router.lqi = () => {throw new Error('failed')};
+        unsupported_router.routingTable = () => {throw new Error('failed')};
+    }
+
+    it('Output raw networkmap', async () => {
+        mock();
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
+
+        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
+
+        const expected = {
+            "nodes":[
+                {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
+                {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
+                {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
+                {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
+            ],
+            "links":[
+                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
+                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
+                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
+                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
+                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
+            ]
+        };
+
+        expect(JSON.parse(call[1])).toStrictEqual(expected);
+
+        /**
+         * Check again without routes
+         */
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap', 'raw');
+        await flushPromises();
+        call = MQTT.publish.mock.calls[0];
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
+
+        // Remove routing information
+        expected.nodes.forEach((n) => {
+            if (n.failed && n.failed.includes('routingTable')) {
+                n.failed.splice(n.failed.indexOf('routingTable'), 1);
+            }
+        });
+
+        expected.links.forEach((l) => l.routes = [])
+        expect(JSON.parse(call[1])).toStrictEqual(expected);
+    });
+
+    it('Output graphviz networkmap', async () => {
+        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
+        const getHours = Date.prototype.getHours;
+        Date.prototype.getTimezoneOffset = () => -60;
+        Date.prototype.getHours = () => 1;
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'graphviz');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/graphviz');
+
+        const expected = `digraph G {
+            node[shape=record];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
+              "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
+            }`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+        Date.prototype.getTimezoneOffset = getTimezoneOffset;
+        Date.prototype.getHours = getHours;
+    });
+});

test/otaUpdate.test.js

@@ -0,0 +1,204 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+describe('OTA update', () => {
+    let controller;
+
+    mockClear = (mapped) => {
+        mapped.ota.updateToLatest = jest.fn();
+        mapped.ota.isUpdateAvailable = jest.fn();
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller();
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+    });
+
+    it('Should subscribe to topics', async () => {
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
+        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+    });
+
+    it('Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10%, +- 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+    });
+
+    it('Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+        logger.info.mockClear();
+        logger.error.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
+    });
+
+    it('Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`No update available for 'bulb'`);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+    });
+
+    it('Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+        logger.error.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
+    });
+
+    it('Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+    });
+
+    it('Should refuse to check/update when already in progress', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
+            return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
+        });
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
+        jest.runAllTimers();
+        await flushPromises();
+    });
+
+    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            if (count === 1) throw new Error('Failed!')
+            count++;
+            return {swBuildId: 1, dateCode: '2019010'}
+        });
+
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+        logger.info.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
+    });
+
+    it('Should check for update when device requests it', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByZigbeeModel(device.modelID)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+
+        // Should not request again when device asks again after a short time
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`)
+    });
+
+    it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
+        const device = zigbeeHerdsman.devices.QBKG04LM;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
+    });
+});

test/settings.test.js

@@ -1,285 +1,641 @@
-const objectAssignDeep = require('object-assign-deep');
-
-const data = require('../lib/util/data');
+require('./stub/logger');
+require('./stub/data');
+const data = require('../lib/util/data.js');
+const utils = require('../lib/util/utils.js');
 const settings = require('../lib/util/settings.js');
-const fs = require('../lib/util/fs');
-
+const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
 const groupsFile = data.joinPath('groups.yaml');
+const secretFile = data.joinPath('secret.yaml');
+const yaml = require('js-yaml');
 
 describe('Settings', () => {
-    const write = (file, json) => fs.writeYaml(file, json);
-
-    const read = (file) => fs.readYaml(file);
-
-    const files = new Map();
-
-    beforeAll(() => {
-        jest.spyOn(fs, 'readYaml').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            throw new Error(`Fake file not found: ${file}`);
-        });
-        jest.spyOn(fs, 'readYamlIfExists').mockImplementation((file) => {
-            if (files.has(file)) return objectAssignDeep.noMutate(files.get(file));
-            return null;
-        });
-        jest.spyOn(fs, 'writeYaml').mockImplementation((file, content) => {
-            files.set(file, objectAssignDeep.noMutate(content));
-        });
-    });
+    const write = (file, json, reread=true) => {
+        fs.writeFileSync(file, yaml.safeDump(json))
+        if (reread) {
+            settings._reRead();
+        }
+    };
+    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const remove = (file) => {
+        if (fs.existsSync(file)) fs.unlinkSync(file);
+    }
 
     beforeEach(() => {
-        settings._clear();
-        files.clear();
+        remove(configurationFile);
+        remove(devicesFile);
+        remove(groupsFile);
     });
 
-    afterAll(() => {
-        fs.readYaml.mockRestore();
-        fs.readYamlIfExists.mockRestore();
-        fs.writeYaml.mockRestore();
+    it('Should return default settings', () => {
+        write(configurationFile, {});
+        const s = settings.get();
+        const expected = settings._getDefaults();
+        expected.devices = {};
+        expected.groups = {};
+        expect(s).toStrictEqual(expected);
     });
 
-    describe('Settings', () => {
-        it('Should return default settings', () => {
-            write(configurationFile, {});
-            const s = settings.get();
-            expect(s).toStrictEqual(settings._getDefaults());
-        });
+    it('Should return settings', () => {
+        write(configurationFile, {permit_join: true});
+        const s = settings.get();
+        const expected = settings._getDefaults();
+        expected.devices = {};
+        expected.groups = {};
+        expected.permit_join = true;
+        expect(s).toStrictEqual(expected);
+    });
 
-        it('Should return settings', () => {
-            write(configurationFile, {permit_join: true});
-            const s = settings.get();
-            const expected = settings._getDefaults();
-            expected.permit_join = true;
-            expect(s).toStrictEqual(expected);
-        });
+    it('Should add devices', () => {
+        write(configurationFile, {});
+        settings.addDevice('0x12345678');
 
-        it('Should add devices', () => {
-            write(configurationFile, {});
-            settings.addDevice('0x12345678');
-
-            const actual = read(configurationFile);
-            const expected = {
-                devices: {
-                    '0x12345678': {
-                        friendly_name: '0x12345678',
-                        retain: false,
-                    },
+        const actual = read(configurationFile);
+        const expected = {
+            devices: {
+                '0x12345678': {
+                    friendly_name: '0x12345678',
                 },
-            };
+            },
+        };
 
-            expect(actual).toStrictEqual(expected);
-        });
+        expect(actual).toStrictEqual(expected);
+    });
 
-        it('Should read devices', () => {
-            const content = {
-                devices: {
-                    '0x12345678': {
-                        friendly_name: '0x12345678',
-                        retain: false,
-                    },
+    it('Should read devices', () => {
+        const content = {
+            devices: {
+                '0x12345678': {
+                    friendly_name: '0x12345678',
+                    retain: false,
                 },
-            };
+            },
+        };
 
-            write(configurationFile, content);
+        write(configurationFile, content);
 
-            const device = settings.getDevice('0x12345678');
-            const expected = {
+        const device = settings.getDevice('0x12345678');
+        const expected = {
+            ID: "0x12345678",
+            friendlyName: "0x12345678",
+            friendly_name: '0x12345678',
+            retain: false,
+        };
+
+        expect(device).toStrictEqual(expected);
+    });
+
+    it('Should not throw error when devices is null', () => {
+        const content = {devices: null};
+        write(configurationFile, content);
+        settings.getDevice('0x12345678');
+    });
+
+    it('Should read MQTT username and password form a separate file', () => {
+        const contentConfiguration = {
+            mqtt: {
+                server: 'my.mqtt.server',
+                user: '!secret username',
+                password: '!secret password',
+            },
+            advanced: {
+                network_key: '!secret network_key'
+            }
+        };
+
+        const contentSecret = {
+            username: 'mysecretusername',
+            password: 'mysecretpassword',
+            network_key: [1,2,3],
+        };
+
+        write(secretFile, contentSecret, false);
+        write(configurationFile, contentConfiguration);
+
+        const expected = {
+            include_device_information: false,
+            password: "mysecretpassword",
+            server: "my.mqtt.server",
+            user: "mysecretusername",
+        };
+
+        expect(settings.get().mqtt).toStrictEqual(expected);
+        expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
+
+        settings._write();
+        expect(read(configurationFile)).toStrictEqual(contentConfiguration);
+    });
+
+    it('Should read devices form a separate file', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        const contentDevices = {
+            '0x12345678': {
                 friendly_name: '0x12345678',
                 retain: false,
-            };
+            },
+        };
 
-            expect(device).toStrictEqual(expected);
-        });
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        const device = settings.getDevice('0x12345678');
+        const expected = {
+            ID: "0x12345678",
+            friendlyName: "0x12345678",
+            friendly_name: '0x12345678',
+            retain: false,
+        };
 
-        it('Should read devices form a separate file', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
+        expect(device).toStrictEqual(expected);
+    });
 
-            const contentDevices = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-            };
+    it('Should add devices to a separate file', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
 
-            write(devicesFile, contentDevices);
-            write(configurationFile, contentConfiguration);
-
-            const device = settings.getDevice('0x12345678');
-            const expected = {
+        const contentDevices = {
+            '0x12345678': {
                 friendly_name: '0x12345678',
                 retain: false,
-            };
+            },
+        };
 
-            expect(device).toStrictEqual(expected);
-        });
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
 
-        it('Should add devices to a separate file', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
+        settings.addDevice('0x1234');
 
-            const contentDevices = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-            };
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
 
-            write(devicesFile, contentDevices);
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            const expected = {
-                '0x12345678': {
-                    friendly_name: '0x12345678',
-                    retain: false,
-                },
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expected);
-        });
-
-        it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
-
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            const expected = {
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expected);
-        }
-        );
-
-        it('Should add and remove devices to a separate file if devices.yaml doesnt exist', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-            };
-
-            write(configurationFile, contentConfiguration);
-
-            settings.addDevice('0x1234');
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            settings.removeDevice('0x1234');
-            expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
-
-            expect(read(devicesFile)).toStrictEqual({});
-        }
-        );
-
-        it('Should read groups', () => {
-            const content = {
-                groups: {
-                    '1': {
-                        friendly_name: '123',
-                    },
-                },
-            };
-
-            write(configurationFile, content);
-
-            const group = settings.getGroup('1');
-            const expected = {
-                friendly_name: '123',
-            };
-
-            expect(group).toStrictEqual(expected);
-        });
-
-        it('Should read groups form a separate file', () => {
-            const contentConfiguration = {
-                groups: 'groups.yaml',
-            };
-
-            const contentGroups = {
-                '1': {
-                    friendly_name: '123',
-                },
-            };
-
-            write(groupsFile, contentGroups);
-            write(configurationFile, contentConfiguration);
-
-            const group = settings.getGroup('1');
-            const expected = {
-                friendly_name: '123',
-            };
-
-            expect(group).toStrictEqual(expected);
-        });
-
-        it('Combine everything! groups and devices from separate file :)', () => {
-            const contentConfiguration = {
-                devices: 'devices.yaml',
-                groups: 'groups.yaml',
-            };
-
-            const contentGroups = {
-                '1': {
-                    friendly_name: '123',
-                },
-            };
-
-            write(groupsFile, contentGroups);
-            write(configurationFile, contentConfiguration);
-
-            const expectedConfiguration = {
-                devices: 'devices.yaml',
-                groups: 'groups.yaml',
-            };
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            settings.addDevice('0x1234');
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            const expectedDevice = {
-                '0x1234': {
-                    friendly_name: '0x1234',
-                    retain: false,
-                },
-            };
-
-            expect(read(devicesFile)).toStrictEqual(expectedDevice);
-
-            const group = settings.getGroup('1');
-            const expectedGroup = {
-                friendly_name: '123',
-            };
-
-            expect(group).toStrictEqual(expectedGroup);
-
-            expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
-
-            const expectedDevice2 = {
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
                 friendly_name: '0x1234',
-                retain: false,
-            };
+            },
+        };
 
-            expect(settings.getDevice('0x1234')).toStrictEqual(expectedDevice2);
+        expect(read(devicesFile)).toStrictEqual(expected);
+    });
+
+    it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        write(configurationFile, contentConfiguration);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        const expected = {
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+    }
+    );
+
+    it('Should add and remove devices to a separate file if devices.yaml doesnt exist', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+        };
+
+        write(configurationFile, contentConfiguration);
+
+        settings.addDevice('0x1234');
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        settings.removeDevice('0x1234');
+        expect(read(configurationFile)).toStrictEqual({devices: 'devices.yaml'});
+
+        expect(read(devicesFile)).toStrictEqual({});
+    }
+    );
+
+    it('Should read groups', () => {
+        const content = {
+            groups: {
+                '1': {
+                    friendly_name: '123',
+                },
+            },
+        };
+
+        write(configurationFile, content);
+
+        const group = settings.getGroup('1');
+        const expected = {
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+            optimistic: true,
+        };
+
+        expect(group).toStrictEqual(expected);
+    });
+
+    it('Should read groups from a separate file', () => {
+        const contentConfiguration = {
+            groups: 'groups.yaml',
+        };
+
+        const contentGroups = {
+            '1': {
+                friendly_name: '123',
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(groupsFile, contentGroups);
+
+        const group = settings.getGroup('1');
+        const expected = {
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+            optimistic: true,
+        };
+
+        expect(group).toStrictEqual(expected);
+    });
+
+    it('Combine everything! groups and devices from separate file :)', () => {
+        const contentConfiguration = {
+            devices: 'devices.yaml',
+            groups: 'groups.yaml',
+        };
+
+        const contentGroups = {
+            '1': {
+                friendly_name: '123',
+                devices: [],
+            },
+        };
+        write(configurationFile, contentConfiguration);
+        write(groupsFile, contentGroups);
+
+        const expectedConfiguration = {
+            devices: 'devices.yaml',
+            groups: 'groups.yaml',
+        };
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        const expectedDevice = {
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expectedDevice);
+
+        const group = settings.getGroup('1');
+        const expectedGroup = {
+            optimistic: true,
+            ID: 1,
+            friendlyName: '123',
+            friendly_name: '123',
+            devices: [],
+        };
+
+        expect(group).toStrictEqual(expectedGroup);
+
+        expect(read(configurationFile)).toStrictEqual(expectedConfiguration);
+
+        const expectedDevice2 = {
+            ID: '0x1234',
+            friendlyName: '0x1234',
+            friendly_name: '0x1234',
+        };
+
+        expect(settings.getDevice('0x1234')).toStrictEqual(expectedDevice2);
+    });
+
+    it('Should add groups', () => {
+        write(configurationFile, {});
+
+        const added = settings.addGroup('test123');
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should add groups with specific ID', () => {
+        write(configurationFile, {});
+
+        const added = settings.addGroup('test123', 123);
+        const expected = {
+            '123': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should not add duplicate groups', () => {
+        write(configurationFile, {});
+
+        settings.addGroup('test123');
+        expect(() => {
+            settings.addGroup('test123');
+        }).toThrow(new Error("friendly_name 'test123' is already in use"));
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should not add duplicate groups with specific ID', () => {
+        write(configurationFile, {});
+
+        settings.addGroup('test123', 123);
+        expect(() => {
+            settings.addGroup('test_id_123', 123);
+        }).toThrow(new Error("group id '123' is already in use"));
+        const expected = {
+            '123': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should add devices to groups', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            }
         });
+
+        settings.addGroup('test123');
+        settings.addDeviceToGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+                devices: ['0x123'],
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should remove devices from groups', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
+                '1': {
+                    friendly_name: 'test123',
+                    devices: ['0x123'],
+                }
+            }
+        });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+                devices: [],
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Shouldnt crash when removing device from group when group has no devices', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+            groups: {
+                '1': {
+                    friendly_name: 'test123',
+                }
+            }
+        });
+
+        settings.removeDeviceFromGroup('test123', ['0x123']);
+        const expected = {
+            '1': {
+                friendly_name: 'test123',
+            },
+        };
+
+        expect(settings.get().groups).toStrictEqual(expected);
+    });
+
+    it('Should throw when adding device to non-existing group', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+        });
+
+        expect(() => {
+            settings.removeDeviceFromGroup('test123', 'bulb')
+        }).toThrow(new Error("Group 'test123' does not exist"));
+    });
+
+    it('Should throw when adding device which already exists', () => {
+        write(configurationFile, {
+            devices: {
+                '0x123': {
+                    friendly_name: 'bulb',
+                    retain: true,
+                }
+            },
+        });
+
+        expect(() => {
+            settings.addDevice('0x123')
+        }).toThrow(new Error("Device '0x123' already exists"));
+    });
+
+    it('Should not allow retention configuration without MQTT v5', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError('MQTT retention requires protocol version 5');
+    });
+
+    it('Should not allow non-existing entities in availability blacklist', () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
+            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+    });
+
+    it('Should ban devices', () => {
+        write(configurationFile, {});
+        settings.banDevice('0x123');
+        expect(settings.get().ban).toStrictEqual(['0x123']);
+        settings.banDevice('0x1234');
+        expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
+    });
+
+    it('Should throw error when yaml file is invalid', () => {
+        fs.writeFileSync(configurationFile, `
+             good: 9
+             \t wrong
+        `)
+
+        let error;
+        try {
+            settings._reRead();
+        } catch (e) {
+            error = e;
+        }
+
+        expect(error.message).toContain(`Your YAML file '${configurationFile}' is invalid`);
+    });
+
+    it('Should throw error when yaml file does not exist', () => {
+        let error;
+        try {
+            settings._reRead();
+        } catch (e) {
+            error = e;
+        }
+
+        expect(error.message).toContain(`no such file or directory, open`);
+    });
+
+    it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname', retain: false}},
+            groups: {'1': {friendly_name: 'myname', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Duplicate friendly_name 'myname' found`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+    });
+
+    it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
+        write(configurationFile, {
+            devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.validate();
+        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
+    });
+
+    it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
+        write(configurationFile, {
+            devices: {
+                '0x0017880104e45519': {friendly_name: 'myname', retain: false},
+                '0x0017880104e45511': {friendly_name: 'myname1', retain: false}
+            },
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.changeFriendlyName('myname1', 'myname');
+        }).toThrowError(`friendly_name 'myname' is already in use`);
+    });
+
+    it('Should throw when removing device which doesnt exist', async () => {
+        write(configurationFile, {
+            devices: {
+                '0x0017880104e45519': {friendly_name: 'myname', retain: false},
+                '0x0017880104e45511': {friendly_name: 'myname1', retain: false}
+            },
+        });
+
+        settings._reRead();
+
+        expect(() => {
+            settings.removeDevice('myname33');
+        }).toThrowError(`Device 'myname33' does not exist`);
+    });
+
+    it('Shouldnt write to configuration.yaml when there are no changes in it', () => {
+        const contentConfiguration = {devices: 'devices.yaml'};
+        const contentDevices = {};
+        write(configurationFile, contentConfiguration);
+        const before = fs.statSync(configurationFile).mtimeMs;
+        write(devicesFile, contentDevices);
+        settings.addDevice('0x1234');
+        const after = fs.statSync(configurationFile).mtimeMs;
+        expect(before).toBe(after);
     });
 });

test/stub/data.js

@@ -0,0 +1,221 @@
+const tmp = require('tmp');
+const yaml = require('../../lib/util/yaml');
+const path = require('path');
+const fs = require('fs');
+
+const mockDir = tmp.dirSync().name;
+const mockDirStorage = tmp.dirSync().name;
+
+function writeDefaultConfiguration() {
+    const config = {
+        homeassistant: false,
+        permit_join: true,
+        mqtt: {
+            base_topic: "zigbee2mqtt",
+            server: "mqtt://localhost",
+        },
+        serial: {
+            "port": "/dev/dummy",
+        },
+        devices: {
+            "0x000b57fffec6a5b2": {
+                retain: true,
+                friendly_name: "bulb"
+            },
+            "0x0017880104e45517": {
+                retain: true,
+                friendly_name: "remote"
+            },
+            "0x0017880104e45520": {
+                retain: false,
+                friendly_name: "button"
+            },
+            "0x0017880104e45521": {
+                retain: false,
+                friendly_name: "button_double_key"
+            },
+            "0x0017880104e45522": {
+                qos: 1,
+                retain: false,
+                friendly_name: "weather_sensor"
+            },
+            "0x0017880104e45523": {
+                retain: false,
+                friendly_name: "occupancy_sensor"
+            },
+            "0x0017880104e45524": {
+                retain: false,
+                friendly_name: "power_plug"
+            },
+            "0x0017880104e45530": {
+                retain: false,
+                friendly_name: "button_double_key_interviewing"
+            },
+            "0x0017880104e45540": {
+                friendly_name: "ikea_onoff"
+            },
+            '0x000b57fffec6a5b7': {
+                retain: false,
+                friendly_name: "bulb_2"
+            },
+            "0x000b57fffec6a5b3": {
+                retain: false,
+                friendly_name: "bulb_color"
+            },
+            '0x000b57fffec6a5b4': {
+                retain: false,
+                friendly_name: "bulb_color_2"
+            },
+            "0x0017880104e45541": {
+                retain: false,
+                friendly_name: "wall_switch"
+            },
+            "0x0017880104e45542": {
+                retain: false,
+                friendly_name: "wall_switch_double"
+            },
+            "0x0017880104e45543": {
+                retain: false,
+                friendly_name: "led_controller_1"
+            },
+            "0x0017880104e45544": {
+                retain: false,
+                friendly_name: "led_controller_2"
+            },
+            '0x0017880104e45545': {
+                retain: false,
+                friendly_name: "dimmer_wall_switch"
+            },
+            '0x0017880104e45547': {
+                retain: false,
+                friendly_name: "curtain"
+            },
+            '0x0017880104e45548': {
+                retain: false,
+                friendly_name: 'fan'
+            },
+            '0x0017880104e45549': {
+                retain: false,
+                friendly_name: 'siren'
+            },
+            '0x0017880104e45529': {
+                retain: false,
+                friendly_name: 'unsupported2'
+            },
+            '0x0017880104e45550': {
+                retain: false,
+                friendly_name: 'thermostat'
+            },
+            '0x0017880104e45551': {
+                retain: false,
+                friendly_name: 'smart_vent'
+            },
+            '0x0017880104e45552': {
+                retain: false,
+                friendly_name: 'j1'
+            },
+            '0x0017880104e45553': {
+                retain: false,
+                friendly_name: 'bulb_enddevice'
+            },
+            '0x0017880104e45559': {
+                retain: false,
+                friendly_name: 'cc2530_router'
+            },
+            '0x0017880104e45560': {
+                retain: false,
+                friendly_name: 'livolo'
+            },
+            '0x90fd9ffffe4b64ae': {
+                retain: false,
+                friendly_name: 'tradfri_remote',
+            },
+            '0x90fd9ffffe4b64af': {
+                friendly_name: 'roller_shutter',
+            },
+            '0x90fd9ffffe4b64ax': {
+                friendly_name: 'ZNLDP12LM',
+            },
+            '0x90fd9ffffe4b64aa': {
+                friendly_name: 'SP600_OLD',
+            },
+            '0x90fd9ffffe4b64ab': {
+                friendly_name: 'SP600_NEW',
+            },
+            '0x90fd9ffffe4b64ac': {
+                friendly_name: 'MKS-CM-W5',
+            },
+            '0x0017880104e45526': {
+                friendly_name: 'GL-S-007ZS',
+            },
+            '0x0017880104e43559': {
+                friendly_name: 'U202DST600ZB'
+            },
+            '0x0017880104e44559': {
+                friendly_name: '3157100_thermostat',
+            }
+        },
+        groups: {
+            '1': {
+                friendly_name: 'group_1',
+                retain: false,
+            },
+            '2': {
+                friendly_name: 'group_2',
+                retain: false,
+            },
+            '15071': {
+                friendly_name: 'group_tradfri_remote',
+                retain: false,
+                devices: ['bulb_color_2', 'bulb_2']
+            },
+            '11': {
+                friendly_name: 'group_with_tradfri',
+                retain: false,
+                devices: ['bulb_2']
+            },
+        }
+    };
+
+    yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
+}
+
+function writeEmptyState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
+}
+
+function writeDefaultState() {
+    const state = {
+        "0x000b57fffec6a5b2": {
+            "state": "ON",
+            "brightness": 50,
+            "color_temp": 370,
+            "linkquality": 99,
+        },
+        "0x0017880104e45517": {
+            "brightness": 255
+        },
+    }
+
+    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
+}
+
+function removeState() {
+    fs.unlinkSync(path.join(mockDir, 'state.json'))
+}
+
+jest.mock('../../lib/util/data', () => ({
+    joinPath: (file) => require('path').join(mockDir, file),
+    getPath: () => mockDir,
+}));
+
+writeDefaultConfiguration();
+writeDefaultState();
+
+module.exports = {
+    mockDir,
+    writeDefaultConfiguration,
+    writeDefaultState,
+    removeState,
+    writeEmptyState,
+};

test/stub/logger.js

@@ -0,0 +1,16 @@
+let level = 'info';
+
+const mock = {
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    debug: jest.fn(),
+    trace: jest.fn(),
+    cleanup: jest.fn(),
+    setLevel: (newLevel) => {level = newLevel},
+    getLevel: () => level,
+};
+
+jest.mock('../../lib/util/logger', () => (mock));
+
+module.exports = {...mock};

test/stub/mqtt.js

@@ -0,0 +1,24 @@
+const events = {};
+
+const mock = {
+    publish: jest.fn().mockImplementation((topic, payload, options, cb) => cb()),
+    end: jest.fn(),
+    subscribe: jest.fn(),
+    on: (type, handler) => {
+        if (type === 'connect') {
+            handler();
+        }
+
+        events[type] = handler
+    },
+};
+
+const mockConnect = jest.fn().mockReturnValue(mock);
+
+jest.mock('mqtt', () => {
+  return {connect: mockConnect};
+});
+
+module.exports = {
+    events, ...mock, connect: mockConnect,
+};

test/stub/zigbeeHerdsman.js

@@ -0,0 +1,203 @@
+const events = {};
+const assert = require('assert');
+
+class Group {
+    constructor(groupID, members) {
+        this.groupID = groupID;
+        this.command = jest.fn();
+        this.meta = {};
+        this.members = members;
+        this.removeFromDatabase = jest.fn();
+        this.hasMember = (endpoint) => this.members.includes(endpoint);
+    }
+}
+
+const clusters = {
+    'genBasic': 0,
+    'genOta': 25,
+    'genScenes': 5,
+    'genOnOff': 6,
+    'genLevelCtrl': 8,
+    'lightingColorCtrl': 768,
+    'closuresWindowCovering': 258,
+}
+
+class Endpoint {
+    constructor(ID, inputClusters, outputClusters, deviceIeeeAddress, binds=[], clusterValues={}) {
+        this.deviceIeeeAddress = deviceIeeeAddress;
+        this.clusterValues = clusterValues;
+        this.ID = ID;
+        this.inputClusters = inputClusters;
+        this.outputClusters = outputClusters;
+        this.command = jest.fn();
+        this.commandResponse = jest.fn();
+        this.read = jest.fn();
+        this.write = jest.fn();
+        this.bind = jest.fn();
+        this.unbind = jest.fn();
+        this.configureReporting = jest.fn();
+        this.binds = binds;
+        this.supportsInputCluster = (cluster) => {
+            assert(clusters[cluster] !== undefined, `Undefined '${cluster}'`);
+            return this.inputClusters.includes(clusters[cluster]);
+        }
+
+        this.supportsOutputCluster = (cluster) => {
+            assert(clusters[cluster], `Undefined '${cluster}'`);
+            return this.outputClusters.includes(clusters[cluster]);
+        }
+
+        this.addToGroup = (group) => {
+            if (!group.members.includes(this)) group.members.push(this);
+        }
+
+        this.getDevice = () => {
+            return Object.values(devices).find(d => d.ieeeAddr === deviceIeeeAddress);
+        }
+
+        this.removeFromGroup = (group) => {
+            const index = group.members.indexOf(this);
+            if (index != -1) {
+                group.members.splice(index, 1);
+            }
+        }
+
+        this.removeFromAllGroups = () => {
+            Object.values(groups).forEach((g) => this.removeFromGroup(g))
+        }
+
+        this.getClusterAttributeValue = (cluster, value) => {
+            return this.clusterValues[cluster][value];
+        }
+    }
+}
+
+class Device {
+    constructor(type, ieeeAddr, networkAddress, manufacturerID, endpoints, interviewCompleted, powerSource = null, modelID = null, interviewing=false, manufacturerName, dateCode= null) {
+        this.type = type;
+        this.ieeeAddr = ieeeAddr;
+        this.dateCode = dateCode;
+        this.networkAddress = networkAddress;
+        this.manufacturerID = manufacturerID;
+        this.endpoints = endpoints;
+        this.powerSource = powerSource;
+        this.interviewCompleted = interviewCompleted;
+        this.modelID = modelID;
+        this.interviewing = interviewing;
+        this.meta = {};
+        this.ping = jest.fn();
+        this.removeFromNetwork = jest.fn();
+        this.removeFromDatabase = jest.fn();
+        this.save = jest.fn();
+        this.manufacturerName = manufacturerName;
+        this.lastSeen = 1000;
+    }
+
+    getEndpoint(ID) {
+        return this.endpoints.find((e) => e.ID === ID);
+    }
+}
+
+const returnDevices = [];
+
+const bulb_color = new Device('Router', '0x000b57fffec6a5b3', 40399, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b3', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "LLC020");
+const bulb_color_2 = new Device('Router', '0x000b57fffec6a5b4', 401292, 4107, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b4')], true, "Mains (single phase)", "LLC020");
+const bulb_2 =  new Device('Router', '0x000b57fffec6a5b7', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b7')], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm");
+
+const devices = {
+    'coordinator': new Device('Coordinator', '0x00124b00120144ae', 0, 0, [new Endpoint(1, [], [])], false),
+    'bulb': new Device('Router', '0x000b57fffec6a5b2', 40369, 4476, [new Endpoint(1, [0,3,4,5,6,8,768,2821,4096], [5,25,32,4096], '0x000b57fffec6a5b2', [], {lightingColorCtrl: {colorCapabilities: 17}})], true, "Mains (single phase)", "TRADFRI bulb E27 WS opal 980lm"),
+    'bulb_color': bulb_color,
+    'bulb_2': bulb_2,
+    'bulb_color_2': bulb_color_2,
+    'remote': new Device('EndDevice', '0x0017880104e45517', 6535, 4107, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x0017880104e45517', [{target: bulb_color.endpoints[0]}]), new Endpoint(2, [0,1,3,15,64512], [25, 6])], true, "Battery", "RWL021"),
+    'unsupported': new Device('EndDevice', '0x0017880104e45518', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'unsupported2': new Device('EndDevice', '0x0017880104e45529', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "notSupportedModelID"),
+    'interviewing': new Device('EndDevice', '0x0017880104e45530', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", undefined, true),
+    'notInSettings': new Device('EndDevice', '0x0017880104e45519', 6537, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
+    'WXKG11LM': new Device('EndDevice', '0x0017880104e45520', 6537,4151, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Battery", "lumi.sensor_switch.aq2"),
+    'WXKG02LM': new Device('EndDevice', '0x0017880104e45521', 6538,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Battery", "lumi.sensor_86sw2.es1"),
+    'WSDCGQ11LM': new Device('EndDevice', '0x0017880104e45522', 6539,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.weather"),
+    'RTCGQ11LM': new Device('EndDevice', '0x0017880104e45523', 6540,4151, [new Endpoint(1, [0], [])], true, "Battery", "lumi.sensor_motion.aq2"),
+    'ZNCZ02LM': new Device('Router', '0x0017880104e45524', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", "lumi.plug"),
+    'E1743': new Device('Router', '0x0017880104e45540', 6540,4476, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'TRADFRI on/off switch'),
+    'QBKG04LM': new Device('Router', '0x0017880104e45541', 6549,4151, [new Endpoint(1, [0], [25]), new Endpoint(2, [0, 6], [])], true, "Mains (single phase)", 'lumi.ctrl_neutral1'),
+    'QBKG03LM':new Device('Router', '0x0017880104e45542', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45542'), new Endpoint(2, [0, 6], [], '0x0017880104e45542'), new Endpoint(3, [0, 6], [], '0x0017880104e45542')], true, "Mains (single phase)", 'lumi.ctrl_neutral2'),
+    'GLEDOPTO1112': new Device('Router', '0x0017880104e45543', 6540, 4151, [new Endpoint(11, [0], [], '0x0017880104e45543'), new Endpoint(13, [0], [], '0x0017880104e45543')], true, "Mains (single phase)", 'GL-C-008'),
+    'GLEDOPTO111213': new Device('Router', '0x0017880104e45544', 6540,4151, [new Endpoint(11, [0], []), new Endpoint(13, [0], []), new Endpoint(12, [0], [])], true, "Mains (single phase)", 'GL-C-008'),
+    'HGZB04D': new Device('Router', '0x0017880104e45545', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'FB56+ZSC05HG1.0'),
+    'ZNCLDJ11LM': new Device('Router', '0x0017880104e45547', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'lumi.curtain'),
+    'HAMPTON99432':  new Device('Router', '0x0017880104e45548', 6540,4151, [new Endpoint(1, [0], []), new Endpoint(2, [0], [])], true, "Mains (single phase)", 'HDC52EastwindFan'),
+    'HS2WD': new Device('Router', '0x0017880104e45549', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'WarningDevice'),
+    '1TST_EU': new Device('Router', '0x0017880104e45550', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'Thermostat'),
+    'SV01': new Device('Router', '0x0017880104e45551', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'SV01-410-MP-1.0'),
+    'J1': new Device('Router', '0x0017880104e45552', 6540,4151, [new Endpoint(1, [0], [])], true, "Mains (single phase)", 'J1 (5502)'),
+    'E11_G13': new Device('EndDevice', '0x0017880104e45553', 6540,4151, [new Endpoint(1, [0, 6], [])], true, "Mains (single phase)", 'E11-G13'),
+    'nomodel': new Device('Router', '0x0017880104e45535', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Mains (single phase)", undefined, true),
+    'unsupported_router': new Device('Router', '0x0017880104e45525', 6536, 0, [new Endpoint(1, [0], [0,3,4,6,8,5])], true, "Mains (single phase)", "notSupportedModelID", false, "Boef"),
+    'CC2530_ROUTER': new Device('Router', '0x0017880104e45559', 6540,4151, [new Endpoint(1, [0, 6], [])], true, "Mains (single phase)", 'lumi.router'),
+    'LIVOLO': new Device('Router', '0x0017880104e45560', 6541,4152, [new Endpoint(6, [0, 6], [])], true, "Mains (single phase)", 'TI0001          '),
+    'tradfri_remote': new Device('EndDevice', '0x90fd9ffffe4b64ae', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64ae')], true, "Battery", "TRADFRI remote control"),
+    'roller_shutter': new Device('EndDevice', '0x90fd9ffffe4b64af', 33906, 4476, [new Endpoint(1, [0], [0,3,4,6,8,5], '0x90fd9ffffe4b64af')], true, "Battery", "SCM-R_00.00.03.15TC"),
+    'ZNLDP12LM': new Device('Router', '0x90fd9ffffe4b64ax', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64ax', [], {lightingColorCtrl: {colorCapabilities: 254}})], true, "Mains (single phase)", "lumi.light.aqcn02"),
+    'SP600_OLD': new Device('Router', '0x90fd9ffffe4b64aa', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20160120'),
+    'SP600_NEW': new Device('Router', '0x90fd9ffffe4b64ab', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {seMetering: {"multiplier":1,"divisor":10000}})], true, "Mains (single phase)", "SP600", false, 'Salus', '20170220'),
+    'MKS-CM-W5': new Device('Router', '0x90fd9ffffe4b64ac', 33901, 4476, [new Endpoint(1, [0,4,3,5,10,258,13,19,6,1,1030,8,768,1027,1029,1026], [0,3,4,6,8,5], '0x90fd9ffffe4b64aa', [], {})], true, "Mains (single phase)", "qnazj70", false),
+    'GL-S-007ZS': new Device('Router', '0x0017880104e45526', 6540,4151, [new Endpoint(1, [0], [], '0x0017880104e45526')], true, "Mains (single phase)", 'GL-S-007ZS'),
+    'U202DST600ZB': new Device('Router', '0x0017880104e43559', 6540,4151, [new Endpoint(10, [0, 6], [], '0x0017880104e43559'), new Endpoint(11, [0, 6], [], '0x0017880104e43559')], true, "Mains (single phase)", 'U202DST600ZB'),
+    '3157100': new Device('Router', '0x0017880104e44559', 6542,4151, [new Endpoint(1, [], [], '0x0017880104e44559')], true, "Mains (single phase)", '3157100'),
+}
+
+const groups = {
+    'group_1': new Group(1, []),
+    'group_tradfri_remote': new Group(15071, [bulb_color_2.endpoints[0], bulb_2.endpoints[0]]),
+    'group/with/slashes': new Group(99, []),
+    'group_with_tradfri': new Group(11, [bulb_2.endpoints[0]])
+}
+
+const mock = {
+    setTransmitPower: jest.fn(),
+    touchlinkFactoryReset: jest.fn(),
+    start: jest.fn(),
+    permitJoin: jest.fn(),
+    getCoordinatorVersion: jest.fn().mockReturnValue({type: 'z-Stack', meta: {version: 1, revision: 20190425}}),
+    getNetworkParameters: jest.fn().mockReturnValue({panID: 0x162a, extendedPanID: [0, 11, 22], channel: 15}),
+    on: (type, handler) => {
+        events[type] = handler;
+    },
+    stop: jest.fn(),
+    setLED: jest.fn(),
+    getDevices: jest.fn().mockImplementation(() => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr));
+    }),
+    getDevicesByType: jest.fn().mockImplementation((type) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).filter((d) => d.type === type);
+    }),
+    getDeviceByIeeeAddr: jest.fn().mockImplementation((ieeeAddr) => {
+        return Object.values(devices).filter((d) => returnDevices.length === 0 || returnDevices.includes(d.ieeeAddr)).find((d) => d.ieeeAddr === ieeeAddr);
+    }),
+    getGroups: jest.fn().mockImplementation((query) => {
+        return Object.values(groups);
+    }),
+    getGroupByID: jest.fn().mockImplementation((groupID) => {
+        return Object.values(groups).find((d) => d.groupID === groupID);
+    }),
+    getPermitJoin: jest.fn().mockReturnValue(false),
+    reset: jest.fn(),
+    createGroup: jest.fn().mockImplementation((groupID) => {
+        const group = new Group(groupID, []);
+        groups[`group_${groupID}`] = group
+        return group;
+    })
+};
+
+const mockConstructor = jest.fn().mockImplementation(() => mock);
+
+jest.mock('zigbee-herdsman', () => ({
+    Controller: mockConstructor,
+    Zcl: {ManufacturerCode: {Philips: 4107}},
+}));
+
+module.exports = {
+    events, ...mock, constructor: mockConstructor, devices, groups, returnDevices
+};

test/utils.js

@@ -1,13 +0,0 @@
-const logger = require('../lib/util/logger');
-
-module.exports = {
-    stubLogger: (jest) => {
-        jest.spyOn(logger, 'info').mockReturnValue(undefined);
-        jest.spyOn(logger, 'warn').mockReturnValue(undefined);
-        jest.spyOn(logger, 'debug').mockReturnValue(undefined);
-        jest.spyOn(logger, 'error').mockReturnValue(undefined);
-    },
-    zigbeeMessage: (device, cid, type, data, epId, groupid=0) => {
-        return {data: {cid, data}, type, groupid, endpoints: [{device, epId}]};
-    },
-};

test/utils.test.js

@@ -0,0 +1,58 @@
+const utils = require('../lib/util/utils.js');
+const version = require('../package.json').version;
+
+describe('Utils', () => {
+    describe('Is xiaomi device', () => {
+        it('Identify xiaomi device', () => {
+            const device = {type: 'Router', manufacturerID: 4151, manufacturerName: 'Xiaomi'};
+            expect(true).toBe(utils.isXiaomiDevice(device));
+        });
+
+        it('Identify xiaomi device without manufacturerName', () => {
+            const device = {type: 'Router', manufacturerID: 4447};
+            expect(true).toBe(utils.isXiaomiDevice(device));
+        });
+
+        it('Identify xiaomi device with different manufacturerName', () => {
+            const device = {type: 'Router', manufacturerID: 4151, manufacturerName: 'Trust International B.V.\u0000'};
+            expect(false).toBe(utils.isXiaomiDevice(device));
+        });
+    });
+
+    it('Convert milliseconds to seconds', () => {
+        expect(utils.millisecondsToSeconds(2000)).toBe(2);
+    })
+
+    it('Object has properties', () => {
+        expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b'])).toBeTruthy();
+        expect(utils.objectHasProperties({a: 1, b: 2, c: 3}, ['a', 'b', 'd'])).toBeFalsy();
+    })
+
+    it('git last commit', async () => {
+        let mockReturnValue = [];
+        jest.mock('git-last-commit', () => ({
+            getLastCommit: (cb) => cb(mockReturnValue[0], mockReturnValue[1])
+        }));
+
+        mockReturnValue = [false, {shortHash: '123'}]
+        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "123", "version": version});
+
+        mockReturnValue = [true, null]
+        expect(await utils.getZigbee2mqttVersion()).toStrictEqual({"commitHash": "unknown", "version": version});
+    })
+
+    it('To local iso string', async () => {
+        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
+        var getTimezoneOffset = Date.prototype.getTimezoneOffset;
+        Date.prototype.getTimezoneOffset = () => 60;
+        expect(utils.formatDate(date, 'ISO_8601_local').endsWith('-01:00')).toBeTruthy();
+        Date.prototype.getTimezoneOffset = () => -60;
+        expect(utils.formatDate(date, 'ISO_8601_local').endsWith('+01:00')).toBeTruthy();
+        Date.prototype.getTimezoneOffset = getTimezoneOffset;
+    })
+
+    it('Throw exception when formating with invalid date', async () => {
+        var date = new Date('August 19, 1975 23:15:30 UTC+00:00');
+        expect(() => utils.formatDate(date, 'invalid', 1)).toThrowError("Unsupported type 'invalid'")
+    })
+});

update.sh

@@ -11,8 +11,7 @@ git checkout HEAD -- npm-shrinkwrap.json
 git pull
 
 echo "Installing dependencies..."
-rm -rf node_modules
-npm install
+npm ci
 
 echo "Restore configuration..."
 cp -R data-backup/* data