1.18.0

* Added support for 16-channels DIY devices.

* Fixed a typo

Co-authored-by: ptvoinfo <xagga@ya.ru>

.dockerignore

@@ -1,7 +1,16 @@
-node_modules
-.travis*
+.eslintignore
+.eslintrc.json
 .git
+.github
 .gitignore
-*.png
-firmware
-docker/Dockerfile*
+.idea
+.npmrc
+.travis*
+.travis.yml
+README.md
+docker/Dockerfile*
+images
+node_modules
+scripts
+test
+update.sh

.eslintignore

@@ -1,2 +1,2 @@
 node_modules/*
-support/*
+test/

.eslintrc.json

@@ -1,17 +1,21 @@
 {
     "env": {
+        "jest/globals": true,
+        "es6": true,
         "node": true
     },
-    "extends": ["eslint:recommended", "google"],
+    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
     "parserOptions": {
-        "ecmaFeatures": {
-            "experimentalObjectRestSpread": true
-        },
+        "ecmaVersion": 2018,
         "sourceType": "module"
     },
     "rules": {
         "require-jsdoc": "off",
         "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }]
-    }
+        "max-len": ["error", { "code": 120 }],
+        "no-prototype-builtins": "off"
+    },
+    "plugins": [
+        "jest"
+    ]
 }

.github/FUNDING.yml

@@ -0,0 +1,2 @@
+github: [koenkk]
+custom: https://www.paypal.me/koenkk

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,46 @@
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+# Bug Report
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug Info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
+Adapter firmware version:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,17 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions/discussion
+    url: https://github.com/Koenkk/zigbee2mqtt/discussions/new
+    about: Ask questions, discuss about devices or show things you made
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
+  - name: Home Assistant addon issues
+    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
+    about: Issues related to the Home Assistant addon.
+  - name: FAQ
+    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    about: Frequently asked questions.
+  - name: Support Chat
+    url: https://discord.gg/NyseBeK
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -0,0 +1,21 @@
+---
+name: New device support request
+about: Request support for a new device
+title: ''
+labels: new device support
+assignees: ''
+
+---
+
+<!--
+VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
+Clearly indicate what you tried and where you got stuck.
+-->
+
+## Information about the device + link
+..
+
+## data/database.db entry of the device
+..

.github/ISSUE_TEMPLATE/problem_report.md

@@ -0,0 +1,54 @@
+---
+name: Problem report
+about: Create a report to help us improve
+title: ''
+labels: problem
+assignees: ''
+
+---
+
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
+Adapter firmware version:

.github/branch-switcher.yml

@@ -0,0 +1 @@
+preferredBranch: dev

.github/stale.yml

@@ -1,17 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-# Issues with these labels will never be considered stale
-exemptLabels:
-  - enhancement
-  - bug
-# Label to use when marking an issue as stale
-staleLabel: stale
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/ci.yml

@@ -0,0 +1,82 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+        registry-url: https://registry.npmjs.org/
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint
+    - name: Docker login
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
+    - name: Docker login ghcr.io
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
+    - name: Docker setup - QEMU
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+    - name: Docker build dev
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
+          .
+    - name: Docker build release
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: |
+        TAG="$(git describe --tags)"
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
+          .
+    - name: Publish to npm
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master, dev ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '28 12 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: "Close stale issues/pull requests"
+on:
+  schedule:
+  - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        days-before-stale: 30
+        days-before-close: 7
+        operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install
+    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
+    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_frontend.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-frontend
+on:
+  repository_dispatch:
+    types: update_frontend
+
+jobs:
+  update_frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
+        branch: "update_frontend"
+        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}

.gitignore

@@ -60,4 +60,17 @@ typings/
 # data
 data/database.db
 data/config.json
-data/log*.txt
+data/log*.txt
+data/state.json
+data/log
+data-backup/
+data/coordinator_backup.json
+data/.storage
+data/database.db.backup
+data/extension
+
+# MacOS indexing file
+.DS_Store
+
+# Ignore config
+data/*.yaml

.npmignore

@@ -0,0 +1,24 @@
+#tests
+test
+coverage
+
+#build tools
+.travis.yml
+.jenkins.yml
+.codeclimate.yml
+.github
+
+#linters
+.jscsrc
+.jshintrc
+.eslintrc*
+.dockerignore
+.eslintignore
+
+#editor settings
+.vscode
+
+#src
+docs
+docker
+images

.travis.yml

@@ -1,28 +0,0 @@
-language: node_js
-
-sudo: required
-
-services:
-  - docker
-
-node_js:
-  - "8"
-
-install:
-  - npm install
-
-script:
-  - npm run verify-homeassistant-mapping
-  - npm run eslint
-
-after_success:
-  - ".travis/docker.sh"
-  - ".travis/wiki.sh"
-  - ".travis/trigger-downstream.sh"
-
-env:
-  global:
-    - DOCKER_USERNAME=koenkk
-    - secure: q5W9+mHz13PslLkTxuYNXivOau+N5OyB53sqgb+krOCPlbN9+gvi2Av9SribVd5jp2yx2O9rV/mr8eS0M2ykvBJ3UsIsCXiFL1dEPmsz1wlFAnW3kbD2EGhGFEV0dWRflSlA+5iQtcMPvbYu2PAOChUUoY7NNlhcz2EC+XXUapvvvaDwA1oSdtX+m61qyREshZb8B57DyXhziy6i/xjhFa84C0mR/iGFRWpZgcxmrNFH+/XDhSIpdq2KshhDWhjyag3ukAsTxTpqzLh1rFRlczjJEn0CcHG4WHsWHUFn9KK2lvT/YCDP8w0URnR1bCr8nyxGZJ2DLW6AzxZtJfWjvbeZYjO2R3WeHpzp6Jw6Qz5pG3gmu2FgeFpZ3FTEGtJ/Ev0/pjkY2Msx9mItzUJ7D8hY2noQZMPASar6oXEDYlkomx7gHHla0MPv3PKSJg5LX6Ybl137GIa6MLYYyz6SEKfyRUQhy3Itqrb/ID/ZP0UuLgyyDfShb8A6XOnsVqwZAsOPtU2UPnD1vsVXhjkpflK9+7QjL5m9MSqfAhg6qtwNsswl1KSiyhmdldGhZW5OfUasDxZKHyx7bsS1BhNnLi/iy4ifDh6KSr64aUW4iM0X99gkU+662Hgf73QpIG+oP41r30iEN0unTcRrFQDIbTSSapbA95bwUCBF1CgQC20=
-    - secure: em3lBVXvZ3kOVI4ipVlh93NIU6guLvyzHMmsLDuxhc92VFURqlYu3iXTTNEnknwyE31mziAkVFnmj+3OENd/VfzP01d4KKaYAQrbn2AnOMvcIIExLfaFU5QqbW1RMwRBIifhvrmboKt07pFsUy3Rghs80SwNSFDfLFoMhFMz+NdIZfJGW8LJy+gUPmz10YGzqh/chreowqOemgBBoswCGwIh1xB1KNejew3bQllnwQYGXcGuTDwDT4Wz+ObxlFt7YR/4n4oJY8jQYRPolM9qHQAGyn/Cn0TL3c8JqiBpVc1hm2OhYSDqvK0xf5IKsD+kIfFJPGa7nF2cwmhKdUNYR47oue3KB5tdrtDhwqUooTglvBOSXUIbthUR9Xu40ziyd5tHy0ixaLnDRGuKIIvEnXlcE6CgApoe3y2kHYa5dY+eUKxWwvH7b5hG+u82dYvw3VYpCHAnDQVgUzz4F63qZ2WZhcH0W3PF3/ytVAHyMu2vAMSrSXGi5xgM6el7Gp7SFVTFXBurbOaoJ3ZwwSEDP5d8mDyk1d8thmkQFL9P6iWtNnXOzjv4GIliEDWQg76BX6MLD3n3fnP0aq+RVmv+ev2Lp4xZZDfgGPek7g35IsaiohH6YvHloxE+ETQ0Ttb1lCQos9fj7LlI+ud6ExjwyRuEnzCOzr/qC/3Leg85Wc8=
-    - secure: GbJVFFY4nmQovp8i2VoQ1AOV8h9QoAcleBV9Mg5TItA8U2Y+6t9wx0iWbaclxOrzI727XZOk5HquMTKKTI430lT6OjU01+YJqwyq+k650YiX5PImsgJXGhFzBKeUcB3A53kKijfr9aEV00YiXRsKoKAn+Bl09m/9O+melBbTwO/YAQyt6EoUt21Yk0BoqpW2CTMPAgsU5tm6Ra3bbJ+IaUYVMYBtuHqtGiKg8jgTHqRhwKejDyV7+1dDxX8q51bK2Kb6PujbI1SS5wsJEpkU/eoPs7MuprmlYarYtBt1fDXSK1EOjcfpyEFRlrNLewIKiFEo3dkTOEtW2a4t3tcAiXOj8Hcvp+6RsjTfnjYmZweHVh/JKDhujjHBMviyTjwp4RnPaQSFou0iVIB5+Mo34hUhFDXc30SBzaux+lyR3rGoEu5pubZNjG5CkKIpw3IyduNKtIrlbCnfg+WDU3jrXmk0fzmwyriyXhx9BAzM+hyFiElZ+3xoXAlmDlwvaRZyJuQQNslwGPbZyX9EByIUkECjYPf92QPyD0mjbn8C32IphgQ515Up+15g2meDniugzowLJkDe3W86M5fB1kPQiDq5oT4M6TgAPRElLN6/ibyfOzDAtOdZgUiQvFnUQb1s9LunZqrc57AKzuzl2xG9Ok/cUkZFSOjrFr40jXgS2Lw=

.travis/docker.sh

@@ -1,43 +0,0 @@
-#!/bin/bash -e
-
-login() {
-  echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-}
-
-tag_push() {
-  docker tag $DOCKER_USERNAME/zigbee2mqtt:$1 $DOCKER_USERNAME/zigbee2mqtt:$2
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$2
-}
-
-build_and_push() {
-  docker build --build-arg COMMIT=$(git rev-parse --short HEAD) -t $DOCKER_USERNAME/zigbee2mqtt:$1 -f $2 .
-  docker push $DOCKER_USERNAME/zigbee2mqtt:$1
-}
-
-# Only update Docker images for:
-# - dev branch
-# - version release
-if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ ! -z ${TRAVIS_TAG+x} ]
-then
-  echo "Updating docker images for master branch!"
-  login
-
-  # Push versioned images
-  build_and_push "$TRAVIS_TAG" docker/Dockerfile.amd64
-  build_and_push "$TRAVIS_TAG-arm32v6" docker/Dockerfile.arm32v6
-  build_and_push "$TRAVIS_TAG-arm64v8" docker/Dockerfile.arm64v8
-
-  # Push latest images.
-  tag_push "$TRAVIS_TAG-arm32v6" "arm32v6"
-  tag_push "$TRAVIS_TAG-arm64v8" "arm64v8"
-  tag_push "$TRAVIS_TAG" latest
-elif [ "$TRAVIS_BRANCH" = "dev" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Updating docker images for dev branch!"
-  login
-  build_and_push latest-dev docker/Dockerfile.amd64
-  build_and_push arm32v6-dev docker/Dockerfile.arm32v6
-  build_and_push arm64v8-dev docker/Dockerfile.arm64v8
-else
-  echo "Not updating docker images, triggered by pull request or not on master/dev branch"
-fi

.travis/trigger-downstream.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-function travis_trigger() {
-     local org=$1 && shift
-     local repo=$1 && shift
-     local branch=${1:-master} && shift
-
-     body="{
-             \"request\": {
-               \"branch\": \"${branch}\"
-              }
-           }"
-
-     curl -s -X POST \
-          -H "Content-Type: application/json" \
-          -H "Accept: application/json" \
-          -H "Travis-API-Version: 3" \
-          -H "Authorization: token $TRAVIS_TOKEN" \
-          -d "$body" \
-          "https://api.travis-ci.org/repo/${org}%2F${repo}/requests"
-}
-
-# Only trigger downstream if on master branch and not pull request
-if [ "$TRAVIS_BRANCH" = "master" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Triggering build of downstream projects!"
-  travis_trigger "danielwelch" "hassio-zigbee2mqtt" "master"
-else
-  echo "Not triggering build of downstream projects, triggered by pull request or not on master branch"
-fi

.travis/wiki.sh

@@ -1,28 +0,0 @@
-#!/bin/sh -e
-
-setup_git() {
-  git config --global user.email "travis@travis-ci.org"
-  git config --global user.name "Travis CI"
-}
-
-update_wiki() {
-  rm -rf zigbee2mqtt.wiki
-  git clone https://${GH_TOKEN}@github.com/Koenkk/zigbee2mqtt.wiki.git
-  npm run docgen zigbee2mqtt.wiki
-  cd zigbee2mqtt.wiki
-  git add -A
-  git diff-index --quiet HEAD || git commit -m "Travis CI: update wiki"
-  git push origin
-  cd ..
-  rm -rf zigbee2mqtt.wiki
-}
-
-# Only update wiki if on master branch and not pull request
-if [ "$TRAVIS_BRANCH" = "master" -a "$TRAVIS_PULL_REQUEST" = "false" ]
-then
-  echo "Updating wiki!"
-  setup_git
-  update_wiki
-else
-  echo "Not updating wiki, triggered by pull request or not on master branch"
-fi

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing to Zigbee2MQTT
+
+Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.
+
+- Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).

README.md

@@ -1,44 +1,78 @@
-[![Travis](https://img.shields.io/travis/Koenkk/zigbee2mqtt.svg)](https://api.travis-ci.org/Koenkk/zigbee2mqtt.svg)
+<div align="center">
+    <a href="https://github.com/koenkk/zigbee2mqtt">
+        <img width="150" height="150" src="images/logo.png">
+    </a>
+    <br>
+    <br>
+    <div style="display: flex;">
+        <a href="https://github.com/Koenkk/zigbee2mqtt/actions?query=workflow%3ACI">
+            <img src="https://github.com/koenkk/zigbee2mqtt/workflows/CI/badge.svg">
+        </a>
+        <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
+            <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
+        </a>
+        <a href="https://github.com/Koenkk/zigbee2mqtt/stargazers">
+            <img src="https://img.shields.io/github/stars/koenkk/zigbee2mqtt.svg">
+        </a>
+        <a href="https://www.paypal.me/koenkk">
+            <img src="https://img.shields.io/badge/donate-PayPal-blue.svg">
+        </a>
+        <a href="https://discord.gg/dadfWYE">
+            <img src="https://img.shields.io/discord/556563650429583360.svg">
+        </a>
+        <a href="http://zigbee2mqtt.discourse.group/">
+            <img src="https://img.shields.io/discourse/https/zigbee2mqtt.discourse.group/status.svg">
+        </a>
+        <a>
+            <img src="https://img.shields.io/badge/Coverage-100%25-brightgreen.svg">
+        </a>
+        <a href="https://www.codacy.com/manual/Koenkk/zigbee2mqtt?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=Koenkk/zigbee2mqtt&amp;utm_campaign=Badge_Grade">
+            <img src="https://api.codacy.com/project/badge/Grade/24f1e0fe39f04daa810e8a1416693d3f">
+        </a>
+        <a href="https://www.npmjs.com/package/zigbee2mqtt">
+            <img src="https://img.shields.io/npm/v/zigbee2mqtt">
+        </a>
+    </div>
+    <h1>Zigbee2MQTT  🌉 🐝</h1>
+    <p>
+        Allows you to use your Zigbee devices <b>without</b> the vendors bridge or gateway.
+    </p>
+    <p>
+        It bridges events and allows you to control your Zigbee devices via MQTT. In this way you can integrate your Zigbee devices with whatever smart home infrastructure you are using.
+    </p>
+</div>
 
-# zigbee2mqtt :bridge_at_night: :honeybee:
-![Thumbs up or down?](header.png)
-<img align="right" height="40px" src="https://user-images.githubusercontent.com/7738048/40915536-1993dd32-6805-11e8-9f22-4dfca52133c0.png">
+## [Getting started](https://www.zigbee2mqtt.io/#getting-started)
+The [documentation](https://www.zigbee2mqtt.io/) provides you all the information needed to get up and running! Make sure you don't skip sections if this is your first visit, as there might be important details in there for you.
 
-Allows you to use your Zigbee devices **without** the vendors (Xiaomi/TRADFRI/Hue) bridge or gateway.
+If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/information/zigbee_network.html) to help you out.
 
-It bridges events and allows you to control your Zigbee devices via MQTT. In this way you can integrate your Zigbee devices with whatever smart home infrastructure you are using. 
-
-## [Getting started](https://github.com/Koenkk/zigbee2mqtt/wiki/Getting-started)
-The [wiki](https://github.com/Koenkk/zigbee2mqtt/wiki) provides you all the information needed to get up and running! Make sure you don't skip sections if this is your first visit, as there might be important details in there for you.
-
-[The documentation](https://github.com/Koenkk/zigbee2mqtt/wiki) will provide information on how to prepare, configure and start **zigbee2mqtt**, but also on how to customize the setup to fit your needs. 
-
-If you aren't familiar with **zigbee** terminology make sure you [read this](https://github.com/Koenkk/zigbee2mqtt/wiki/ZigBee-network) to help you out.
-
-## Integrations
-Zigbee2mqtt integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
+## [Integrations](https://www.zigbee2mqtt.io/#integration)
+Zigbee2MQTT integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Hassio](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
-- Generic install or Hassbian: Using instructions [here](https://github.com/Koenkk/zigbee2mqtt/wiki/Integrating-with-Home-Assistant)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
+- Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
-## Architecture 
-![Architecture](architecture.png)
+<img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
+
+### [Domoticz](https://www.domoticz.com/)
+- Integration implemented in [domoticz-zigbee2mqtt-plugin](https://github.com/stas-demydiuk/domoticz-zigbee2mqtt-plugin)
+
+<br>
+
+## Architecture
+![Architecture](images/architecture.png)
+
+### Internal Architecture
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
 
 ## Supported devices
-See [Supported devices](https://github.com/Koenkk/zigbee2mqtt/wiki/Supported-devices) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more. 
+See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.
 
-If it's not listed in [Supported devices](https://github.com/Koenkk/zigbee2mqtt/wiki/Supported-devices), support can be added (fairly) easy, see [How to support new devices](https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices).
+If it's not listed in [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html), support can be added (fairly) easy, see [How to support new devices](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).
 
 ## Support & help
-If you need assistance you can check [opened issues](https://github.com/Koenkk/zigbee2mqtt/issues). Feel free to help with Pull Requests when you were able to fix things or add new devices or just share the love on social media. 
-
-## Contributors
-* [AndrewLinden](https://github.com/AndrewLinden)
-* [oskarn97](https://github.com/oskarn97)
-* [dgomes](https://github.com/dgomes)
-* [Koenkk](https://github.com/Koenk)
-* [kirovilya](https://github.com/kirovilya)
-* [ciotlosm](https://github.com/ciotlosm)
+If you need assistance you can check [opened issues](https://github.com/Koenkk/zigbee2mqtt/issues). Feel free to help with Pull Requests when you were able to fix things or add new devices or just share the love on social media.

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -0,0 +1,33 @@
+FROM node:14-alpine3.12 as base
+
+WORKDIR /app
+RUN apk add --no-cache tzdata eudev
+
+COPY package.json ./
+
+# Dependencies
+FROM base as dependencies
+
+COPY npm-shrinkwrap.json ./
+
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
+    npm ci --production && \
+    apk del .buildtools
+
+# Release
+FROM base as release
+
+COPY --from=dependencies /app/node_modules ./node_modules
+COPY lib ./lib
+COPY LICENSE index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
+
+RUN mkdir /app/data
+
+ARG COMMIT
+RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["npm", "start"]

docker/Dockerfile.amd64

@@ -1,20 +0,0 @@
-FROM alpine:3.7
-
-# Copy files
-ADD . /app
-RUN cp /app/data/configuration.yaml /app
-RUN cp /app/docker/run.sh /app
-RUN chmod +x /app/run.sh
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
-
-# Install dependencies
-RUN apk add --update --no-cache make gcc g++ python linux-headers udev nodejs git && \
-    npm install --unsafe-perm && \
-    apk del make gcc g++ python linux-headers udev git
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm32v6

@@ -1,22 +0,0 @@
-FROM arm32v6/alpine:3.7
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-arm-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-arm-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-arm-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/Dockerfile.arm64v8

@@ -1,22 +0,0 @@
-FROM arm64v8/alpine:3.7
-
-# Setup cross-build
-ENV QEMU_EXECVE 1
-COPY docker/qemu-aarch64-static /usr/bin
-WORKDIR /app
-
-# Write .hash.json
-ARG COMMIT
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", "echo {\\\"hash\\\": \\\"$COMMIT\\\"} > .hash.json" ]
-
-# Copy files & install dependencies
-ADD . /app
-RUN [ "qemu-aarch64-static", "/bin/sh", "-c", \
-      "cp /app/data/configuration.yaml /app && \
-       cp /app/docker/run.sh /app && chmod +x /app/run.sh && \
-       apk add --update --no-cache make gcc g++ python linux-headers udev nodejs git && \
-       npm install --unsafe-perm && \
-       apk del make gcc g++ python linux-headers udev git"]
-
-# Entrypoint
-ENTRYPOINT ["./run.sh"]

docker/docker-compose.yml

@@ -0,0 +1,14 @@
+  version: '3'
+  services:
+    zigbee2mqtt:
+      container_name: zigbee2mqtt
+      image: koenkk/zigbee2mqtt
+      volumes:
+        - ./data:/app/data
+      devices:
+        # CC251
+        #- /dev/ttyUSB_cc2531:/dev/ttyACM0
+        # CC2530 / GBAN GB2530S
+        #- /dev/ttyUSB_cc2530:/dev/ttyACM0
+      restart: always
+      network_mode: host

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,8 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-if [ ! -z "$DEBUG" ]; then
-    npm start
-else
-    DEBUG="$DEBUG" npm start
-fi
+exec "$@"

index.js

@@ -1,10 +1,59 @@
-const Controller = require('./lib/controller');
+const semver = require('semver');
+const engines = require('./package.json').engines;
+const indexJsRestart = 'indexjs.restart';
 
-const controller = new Controller();
-controller.start();
+let controller;
+let stopping = false;
+
+async function restart() {
+    await stop(indexJsRestart);
+    await start();
+}
+
+async function exit(code, reason) {
+    if (reason !== indexJsRestart) {
+        process.exit(code);
+    }
+}
+
+async function start() {
+    const version = engines.node;
+    if (!semver.satisfies(process.version, version)) {
+        console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    }
+
+    // Validate settings
+    const settings = require('./lib/util/settings');
+    const errors = settings.validate();
+    if (errors.length > 0) {
+        console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+        console.log('            READ THIS CAREFULLY\n');
+        console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+        for (const error of errors) {
+            console.log(`- ${error}`);
+        }
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+        exit(1);
+    }
+
+    const Controller = require('./lib/controller');
+    controller = new Controller(restart, exit);
+    await controller.start();
+}
+
+async function stop(reason=null) {
+    await controller.stop(reason);
+}
+
+async function handleQuit() {
+    if (!stopping && controller) {
+        stopping = true;
+        await stop();
+    }
+}
 
 process.on('SIGINT', handleQuit);
+process.on('SIGTERM', handleQuit);
 
-function handleQuit() {
-    controller.stop(() => process.exit());
-}
+start();

lib/controller.js

@@ -1,552 +1,393 @@
 const MQTT = require('./mqtt');
 const Zigbee = require('./zigbee');
+const EventBus = require('./eventBus');
+const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const zigbeeShepherdConverters = require('zigbee-shepherd-converters');
-const homeassistant = require('./homeassistant');
-const objectAssignDeep = require(`object-assign-deep`);
+const objectAssignDeep = require('object-assign-deep');
+const utils = require('./util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+const assert = require('assert');
 
-const mqttConfigRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/\\w+`, 'g');
-const mqttDeviceRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d]+/set`, 'g');
-const mqttDevicePrefixRegex = new RegExp(`${settings.get().mqtt.base_topic}/[\\w\\s\\d]+/[\\w\\s\\d]+/set`, 'g');
+// Extensions
+const ExtensionFrontend = require('./extension/frontend');
+const ExtensionPublish = require('./extension/publish');
+const ExtensionReceive = require('./extension/receive');
+const ExtensionNetworkMap = require('./extension/networkMap');
+const ExtensionSoftReset = require('./extension/legacy/softReset');
+const ExtensionHomeAssistant = require('./extension/homeassistant');
+const ExtensionConfigure = require('./extension/configure');
+const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
+const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
+const ExtensionBridge = require('./extension/bridge');
+const ExtensionGroups = require('./extension/groups');
+const ExtensionAvailability = require('./extension/availability');
+const ExtensionBind = require('./extension/bind');
+const ExtensionReport = require('./extension/legacy/report');
+const ExtensionOnEvent = require('./extension/onEvent');
+const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
+const ExtensionExternalExtension = require('./extension/externalExtension');
 
-const pollInterval = 60 * 1000; // seconds * 1000.
-const softResetTimeout = 3600 * 1000; // seconds * 1000.
-
-const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
-
-/**
- * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
- * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
- *
- * Therefore zigbee2mqtt BY DEFAULT caches all values and resend it with every message.
- * advanced.cache_state in configuration.yaml allows to configure this.
- * https://github.com/Koenkk/zigbee2mqtt/wiki/Configuration
- */
-const cacheState = settings.get().advanced && settings.get().advanced.cache_state === false ? false : true;
-if (settings.get().homeassistant && !cacheState) {
-    logger.warn('In order for Home Assistant integration to work properly set `cache_state: true');
-}
+const AllExtensions = [
+    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
+    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
+    ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension,
+];
 
 class Controller {
-    constructor() {
+    constructor(restartCallback, exitCallback) {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
-        this.stateCache = {};
-        this.configured = [];
-        this.handleZigbeeMessage = this.handleZigbeeMessage.bind(this);
-        this.handleMQTTMessage = this.handleMQTTMessage.bind(this);
-    }
+        this.eventBus = new EventBus();
+        this.state = new State(this.eventBus);
+        this.restartCallback = restartCallback;
+        this.exitCallback = exitCallback;
 
-    start() {
-        this.startupLogVersion(() => {
-            this.zigbee.start(this.handleZigbeeMessage, (error) => {
-                if (error) {
-                    logger.error('Failed to start', error);
-                } else {
-                    // Log zigbee clients on startup and configure.
-                    const devices = this.zigbee.getAllClients();
-                    logger.info(`Currently ${devices.length} devices are joined:`);
-                    devices.forEach((device) => {
-                        logger.info(this.getDeviceStartupLogMessage(device));
-                        this.configureDevice(device);
-                    });
+        this.publishEntityState = this.publishEntityState.bind(this);
+        this.enableDisableExtension = this.enableDisableExtension.bind(this);
+        this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
+        this.addExtension = this.addExtension.bind(this);
 
-                    // Enable zigbee join.
-                    if (settings.get().permit_join) {
-                        logger.warn('`permit_join` set to  `true` in configuration.yaml.');
-                        logger.warn('Allowing new devices to join.');
-                        logger.warn('Set `permit_join` to `false` once you joined all devices.');
-                        this.zigbee.permitJoin(true);
-                    }
+        // Initialize extensions.
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
+        this.extensions = [
+            new ExtensionBridge(...args, this.enableDisableExtension, this.restartCallback),
+            new ExtensionPublish(...args),
+            new ExtensionReceive(...args),
+            new ExtensionDeviceGroupMembership(...args),
+            new ExtensionConfigure(...args),
+            new ExtensionNetworkMap(...args),
+            new ExtensionGroups(...args),
+            new ExtensionBind(...args),
+            new ExtensionOnEvent(...args),
+            new ExtensionOTAUpdate(...args),
+            new ExtensionReport(...args),
+        ];
 
-                    // Start timers.
-                    this.pollTimer(true);
-                    this.softResetTimeout(true);
+        if (settings.get().frontend) {
+            this.extensions.push(new ExtensionFrontend(...args));
+        }
 
-                    // Connect to MQTT broker
-                    const subscriptions = [
-                        `${settings.get().mqtt.base_topic}/+/set`,
-                        `${settings.get().mqtt.base_topic}/+/+/set`,
-                        `${settings.get().mqtt.base_topic}/bridge/config/+`,
-                    ];
+        if (settings.get().advanced.legacy_api) {
+            this.extensions.push(new ExtensionBridgeLegacy(...args));
+        }
 
-                    if (settings.get().homeassistant) {
-                        subscriptions.push('hass/status');
-                    }
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
+        }
 
-                    this.mqtt.connect(this.handleMQTTMessage, subscriptions, () => this.handleMQTTConnected());
-                }
-            });
-        });
-    }
-
-    handleMQTTConnected() {
-        // Home Assistant MQTT discovery on MQTT connected.
         if (settings.get().homeassistant) {
-            // MQTT discovery of all paired devices on startup.
-            this.zigbee.getAllClients().forEach((device) => {
-                const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                if (mappedModel) {
-                    homeassistant.discover(device.ieeeAddr, mappedModel.model, this.mqtt, true);
-                }
-            });
+            this.extensions.push(new ExtensionHomeAssistant(...args));
         }
 
-        // Resend all cached states.
-        this.sendAllCachedStates();
+        /* istanbul ignore next */
+        if (settings.get().advanced.soft_reset_timeout !== 0) {
+            this.extensions.push(new ExtensionSoftReset(...args));
+        }
+
+        if (settings.get().advanced.availability_timeout) {
+            this.extensions.push(new ExtensionAvailability(...args));
+        }
+        this.extensions.push(new ExtensionExternalExtension(...args, this.addExtension, this.enableDisableExtension));
     }
 
-    sendAllCachedStates() {
-        this.zigbee.getAllClients().forEach((device) => {
-            if (this.stateCache.hasOwnProperty(device.ieeeAddr)) {
-                this.mqttPublishDeviceState(device.ieeeAddr, this.stateCache[device.ieeeAddr], false);
-            }
-        });
-    }
+    async start() {
+        this.state.start();
+        logger.logOutput();
 
-    softResetTimeout(start) {
-        if (this._softResetTimer) {
-            clearTimeout(this._softResetTimer);
-            this._softResetTimer = null;
-        }
+        const info = await utils.getZigbee2mqttVersion();
+        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
-        if (start) {
-            this._softResetTimer = setTimeout(() => {
-                this.zigbee.softReset((error) => {
-                    if (error) {
-                        logger.warn('Soft reset error', error);
-                        this.zigbee.stop((error) => {
-                            logger.warn('Zigbee stopped');
-                            this.zigbee.start(this.handleZigbeeMessage, (error) => {
-                                if (error) {
-                                    logger.error('Failed to restart!');
-                                }
-                            });
-                        });
-                    } else {
-                        logger.warn('Soft resetted zigbee');
-                    }
-
-                    this.softResetTimeout(true);
-                });
-            }, softResetTimeout);
-        }
-    }
-
-    pollTimer(start) {
-        // Some routers need polling to prevent them from sleeping.
-        if (start && !this._pollTimer) {
-            this._pollTimer = setInterval(() => {
-                const devices = this.zigbee.getAllClients().filter((d) => {
-                    const power = d.powerSource ? d.powerSource.toLowerCase().split(' ')[0] : 'unknown';
-                    return power !== 'battery' && power !== 'unknown' && d.type === 'Router';
-                });
-
-                devices.forEach((d) => this.zigbee.ping(d.ieeeAddr));
-            }, pollInterval);
-        } else if (!start && this._pollTimer) {
-            clearTimeout(this._pollTimer);
-            this._pollTimer = null;
-        }
-    }
-
-    stop(callback) {
-        this.mqtt.disconnect();
-        this.pollTimer(false);
-        this.softResetTimeout(false);
-        this.zigbee.stop(callback);
-    }
-
-    configureDevice(device) {
-        const ieeeAddr = device.ieeeAddr;
-        if (ieeeAddr && device.modelId && !this.configured.includes(ieeeAddr)) {
-            const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-
-            // Call configure function of device.
-            if (mappedModel && mappedModel.configure) {
-                mappedModel.configure(ieeeAddr, this.zigbee.shepherd, this.zigbee.getCoordinator(), (ok, msg) => {
-                    if (ok) {
-                        logger.info(`Succesfully configured ${ieeeAddr}`);
-                    } else {
-                        logger.error(`Failed to configure ${ieeeAddr}`);
-                    }
-                });
-            }
-
-            // Setup an OnAfIncomingMsg handler if needed.
-            if (mappedModel && mappedModel.onAfIncomingMsg) {
-                mappedModel.onAfIncomingMsg.forEach((ep) => this.zigbee.registerOnAfIncomingMsg(ieeeAddr, ep));
-            }
-
-            this.configured.push(ieeeAddr);
-        }
-    }
-
-    getDeviceStartupLogMessage(device) {
-        let friendlyName = 'unknown';
-        let type = 'unknown';
-        let friendlyDevice = {model: 'unkown', description: 'unknown'};
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        if (mappedModel) {
-            friendlyDevice = mappedModel;
-        }
-
-        if (settings.getDevice(device.ieeeAddr)) {
-            friendlyName = settings.getDevice(device.ieeeAddr).friendly_name;
-        }
-
-        if (device.type) {
-            type = device.type;
-        }
-
-        return `${friendlyName} (${device.ieeeAddr}): ${friendlyDevice.model} - ` +
-            `${friendlyDevice.vendor} ${friendlyDevice.description} (${type})`;
-    }
-
-    handleZigbeeMessage(message) {
-        // Zigbee message receieved, reset soft reset timeout.
-        this.softResetTimeout(true);
-
-        logger.debug('Recieved zigbee message with data', JSON.stringify(message.data));
-        if (message.type == 'devInterview' && !settings.getDevice(message.data)) {
-            logger.info('Connecting with device...');
-            this.mqtt.log('pairing', 'connecting with device');
-        }
-
-        if (message.type == 'devIncoming') {
-            logger.info('Device incoming...');
-            this.mqtt.log('pairing', 'device incoming');
-        }
-
-        // We dont handle messages without endpoints.
-        if (!message.endpoints) {
-            return;
-        }
-
-        const device = message.endpoints[0].device;
-
-        if (!device) {
-            logger.warn('Message without device!');
-            return;
-        }
-
-        // Check if this is a new device.
-        if (!settings.getDevice(device.ieeeAddr)) {
-            logger.info(`New device with address ${device.ieeeAddr} connected!`);
-            settings.addDevice(device.ieeeAddr);
-            this.mqtt.log('device_connected', device.ieeeAddr);
-        }
-
-        // We can't handle devices without modelId.
-        if (!device.modelId) {
-            return;
-        }
-
-        // Map Zigbee modelID to vendor modelID.
-        const modelID = message.endpoints[0].device.modelId;
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(modelID);
-
-        if (!mappedModel) {
-            logger.warn(`Device with modelID '${modelID}' is not supported.`);
-            logger.warn(`Please see: https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices`);
-            return;
-        }
-
-        // Configure device.
-        this.configureDevice(device);
-
-        // Home Assistant MQTT discovery
-        if (settings.get().homeassistant) {
-            homeassistant.discover(device.ieeeAddr, mappedModel.model, this.mqtt, false);
-        }
-
-        // After this point we cant handle message withoud cid anymore.
-        if (!message.data || (!message.data.cid && !message.data.cmdId)) {
-            return;
-        }
-
-        // Find a conveter for this message.
-        const cid = message.data.cid;
-        const cmdId = message.data.cmdId;
-        const converters = mappedModel.fromZigbee.filter((c) => {
-            if (cid) {
-                return c.cid === cid && c.type === message.type;
-            } else if (cmdId) {
-                return c.cmd === cmdId;
-            }
-
-            return false;
-        });
-
-        if (!converters.length) {
-            if (cid) {
-                logger.warn(
-                    `No converter available for '${mappedModel.model}' with cid '${cid}' and type '${message.type}'`
-                );
-            } else if (cmdId) {
-                logger.warn(`No converter available for '${mappedModel.model}' with cmd '${cmdId}'`);
-            }
-
-            logger.warn(`Please see: https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices.`);
-            return;
-        }
-
-        // Convert this Zigbee message to a MQTT message.
-        // Get payload for the message.
-        // - If a payload is returned publish it to the MQTT broker
-        // - If NO payload is returned do nothing. This is for non-standard behaviour
-        //   for e.g. click switches where we need to count number of clicks and detect long presses.
-        converters.forEach((converter) => {
-            const publish = (payload) => {
-                // Don't cache messages with following properties:
-                const dontCacheProperties = ['click', 'action', 'button', 'button_left', 'button_right'];
-                let cache = true;
-                dontCacheProperties.forEach((property) => {
-                    if (payload.hasOwnProperty(property)) {
-                        cache = false;
-                    }
-                });
-
-                this.mqttPublishDeviceState(device.ieeeAddr, payload, cache);
-            };
-
-            const payload = converter.convert(mappedModel, message, publish, settings.getDevice(device.ieeeAddr));
-
-            if (payload) {
-                publish(payload);
-            }
-        });
-    }
-
-    handleMQTTMessage(topic, message) {
-        logger.debug(`Recieved mqtt message on topic '${topic}' with data '${message}'`);
-
-        if (topic.match(mqttConfigRegex)) {
-            this.handleMQTTMessageConfig(topic, message);
-        } else if (topic.match(mqttDeviceRegex) || topic.match(mqttDevicePrefixRegex)) {
-            this.handleMQTTMessageDevice(topic, message, topic.match(mqttDevicePrefixRegex));
-        } else if (topic === 'hass/status') {
-            if (message.toString().toLowerCase() === 'online') {
-                const timer = setTimeout(() => {
-                    this.sendAllCachedStates();
-                    clearTimeout(timer);
-                }, 20000);
-            }
-        } else {
-            logger.warn(`Cannot handle MQTT message with topic '${topic}' and message '${message}'`);
-        }
-    }
-
-    handleMQTTMessageConfig(topic, message) {
-        const option = topic.split('/')[3];
-
-        if (option === 'permit_join') {
-            this.zigbee.permitJoin(message.toString().toLowerCase() === 'true');
-        } else if (option === 'log_level') {
-            const level = message.toString().toLowerCase();
-            if (allowedLogLevels.includes(level)) {
-                logger.info(`Switching log level to '${level}'`);
-                logger.transports.console.level = level;
-                logger.transports.file.level = level;
-            } else {
-                logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
-            }
-        } else if (option === 'devices') {
-            const devices = this.zigbee.getAllClients().map((device) => {
-                const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-
-                return {
-                    ieeeAddr: device.ieeeAddr,
-                    type: device.type,
-                    model: mappedModel ? mappedModel.model : device.modelId,
-                    friendly_name: friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr,
-                };
-            });
-
-            this.mqtt.log('devices', devices);
-        } else if (option === 'remove') {
-            message = message.toString();
-            const IDByFriendlyName = settings.getIDByFriendlyName(message);
-            const deviceID = IDByFriendlyName ? IDByFriendlyName : message;
-            const device = this.zigbee.getDevice(deviceID);
-
-            const cleanup = () => {
-                // Clear Home Assistant MQTT discovery message
-                if (settings.get().homeassistant && device) {
-                    const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-                    if (mappedModel) {
-                        homeassistant.clear(deviceID, mappedModel.model, this.mqtt);
-                    }
-                }
-
-                // Remove from configuration.yaml
-                settings.removeDevice(deviceID);
-
-                logger.info(`Successfully removed ${deviceID}`);
-                this.mqtt.log('device_removed', message);
-            };
-
-            // Remove from zigbee network.
-            if (device) {
-                this.zigbee.removeDevice(deviceID, (error) => {
-                    if (!error) {
-                        cleanup();
-                    } else {
-                        logger.error(`Failed to remove ${deviceID}`);
-                    }
-                });
-            } else {
-                cleanup();
-            }
-        } else {
-            logger.warn(`Cannot handle MQTT config option '${option}' with message '${message}'`);
-        }
-    }
-
-    handleMQTTMessageDevice(topic, message, withPrefix) {
-        const friendlyName = topic.split('/')[1];
-        const topicPrefix = withPrefix ? topic.split('/')[2] : '';
-
-        // Map friendlyName to deviceID.
-        const deviceID = settings.getIDByFriendlyName(friendlyName);
-
-        if (!deviceID) {
-            logger.error(`Cannot handle '${topic}' because deviceID of '${friendlyName}' cannot be found`);
-            return;
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = null;
+        // Start zigbee
         try {
-            json = JSON.parse(message);
-        } catch (e) {
-            // Cannot be parsed to JSON, assume state message.
-            json = {state: message.toString()};
+            await this.zigbee.start();
+            this.callExtensionMethod('onZigbeeStarted', []);
+            this.zigbee.on('event', this.onZigbeeEvent.bind(this));
+            this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
+        } catch (error) {
+            logger.error('Failed to start zigbee');
+            logger.error('Exiting...');
+            logger.error(error.stack);
+            this.exitCallback(1);
         }
 
-        // Find ep for this device
-        const device = this.zigbee.getDevice(deviceID);
-        if (!device) {
-            logger.error(`Failed to find device with deviceID ${deviceID}`);
-            return;
+        // Log zigbee clients on startup
+        const devices = this.zigbee.getClients();
+        logger.info(`Currently ${devices.length} devices are joined:`);
+        for (const device of devices) {
+            const entity = this.zigbee.resolveEntity(device);
+            const model = entity.definition ?
+                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
+                'Not supported';
+            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
         }
 
-        const mappedModel = zigbeeShepherdConverters.findByZigbeeModel(device.modelId);
-        if (!mappedModel) {
-            logger.warn(`Device with modelID '${device.modelId}' is not supported.`);
-            logger.warn(`Please see: https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices`);
-            return;
+        // Enable zigbee join.
+        if (settings.get().permit_join) {
+            logger.warn('`permit_join` set to  `true` in configuration.yaml.');
+            logger.warn('Allowing new devices to join.');
+            logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        const ep = mappedModel.ep && mappedModel.ep[topicPrefix] ? mappedModel.ep[topicPrefix] : null;
-        const published = [];
+        try {
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
 
-        Object.keys(json).forEach((key) => {
-            // Find converter for this key.
-            const converter = mappedModel.toZigbee.find((c) => c.key === key);
+        // MQTT
+        this.mqtt.on('message', this.onMQTTMessage.bind(this));
+        await this.mqtt.connect();
 
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                return;
-            }
-
-            const message = converter.convert(json[key], json);
-
-            if (!message) {
-                return;
-            }
-
-            const callback = (error) => {
-                // Devices do not report when they go off, this ensures state (on/off) is always in sync.
-                if (!error && key.startsWith('state')) {
-                    const msg = {};
-                    const _key = topicPrefix ? `${key}_${topicPrefix}` : key;
-                    msg[_key] = json[key];
-                    this.mqttPublishDeviceState(deviceID, msg, true);
+        // Send all cached states.
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
+            for (const device of this.zigbee.getClients()) {
+                if (this.state.exists(device.ieeeAddr)) {
+                    this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
                 }
-            };
-
-            this.zigbee.publish(deviceID, message.cid, message.cmd, message.zclData, ep, callback);
-
-            published.push({message: message, converter: converter});
-        });
-
-        /**
-         * After publishing a command to a zigbee device we want to monitor the changed attribute(s) so that
-         * everything stays in sync.
-         */
-        published.forEach((p) => {
-            let counter = 0;
-            let secondsToMonitor = 1;
-
-            // In case of a transition we need to monitor for the whole transition time.
-            if (p.message.zclData.hasOwnProperty('transtime')) {
-                // Note that: transtime 10 = 0.1 seconds, 100 = 1 seconds, etc.
-                secondsToMonitor = (p.message.zclData.transtime / 10) + 1;
             }
+        }
 
-            const timer = setInterval(() => {
-                counter++;
+        // Add devices which are in database but not in configuration to configuration
+        for (const device of this.zigbee.getClients()) {
+            if (!settings.getDevice(device.ieeeAddr)) {
+                settings.addDevice(device.ieeeAddr);
+            }
+        }
 
-                // Doing a 'read' will result in the device sending a zigbee message with the current attribute value.
-                // which will be handled by this.handleZigbeeMessage.
-                p.converter.attr.forEach((attribute) => {
-                    this.zigbee.read(deviceID, p.message.cid, attribute, ep, () => null);
-                });
-
-                if (counter >= secondsToMonitor) {
-                    clearTimeout(timer);
-                }
-            }, 1000);
-        });
+        // Call extensions
+        await this.callExtensionMethod('onMQTTConnected', []);
     }
 
-    mqttPublishDeviceState(deviceID, payload, cache) {
-        if (cacheState) {
-            // Add cached state to payload
-            if (this.stateCache[deviceID]) {
-                payload = objectAssignDeep.noMutate(this.stateCache[deviceID], payload);
+    async enableDisableExtension(enable, name) {
+        if (!enable) {
+            const extension = this.extensions.find((e) => e.constructor.name === name);
+            if (extension) {
+                await this.callExtensionMethod('stop', [], [extension]);
+                this.extensions.splice(this.extensions.indexOf(extension), 1);
             }
+        } else {
+            const Extension = AllExtensions.find((e) => e.name === name);
+            assert(Extension, `Extension '${name}' does not exist`);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
+            this.extensions.push(extension);
+            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            this.callExtensionMethod('onMQTTConnected', [], [extension]);
+        }
+    }
 
-            // Update state cache with new state.
-            if (cache) {
-                this.stateCache[deviceID] = payload;
+    addExtension(extension) {
+        this.extensions.push(extension);
+    }
+
+    async stop(reason=null) {
+        // Call extensions
+        await this.callExtensionMethod('stop', []);
+
+        // Wrap-up
+        this.state.stop();
+        await this.mqtt.disconnect();
+
+        try {
+            await this.zigbee.stop();
+            logger.info('Stopped Zigbee2MQTT');
+            this.exitCallback(0, reason);
+        } catch (error) {
+            logger.error('Failed to stop Zigbee2MQTT');
+            this.exitCallback(1, reason);
+        }
+    }
+
+    async onZigbeeAdapterDisconnected() {
+        logger.error('Adapter disconnected, stopping');
+        await this.stop();
+    }
+
+    async onZigbeeEvent(type, data) {
+        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
+        }
+
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
+            // Only deviceLeave doesn't have a device (not interesting to add to settings)
+            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
+        }
+
+        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
+
+        if (type === 'message') {
+            logger.debug(
+                `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
+                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
+            );
+        } else if (type === 'deviceJoined') {
+            logger.info(`Device '${name}' joined`);
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
+
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
+                } else {
+                    logger.warn(
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
+                        `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
+                    );
+                }
+            } else if (data.status === 'failed') {
+                logger.error(`Failed to interview '${name}', device has not successfully been paired`);
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    logger.info(`Starting interview of '${name}'`);
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            logger.debug(`Device '${name}' announced itself`);
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                this.state.remove(data.ieeeAddr);
             }
         }
 
-        const deviceSettings = settings.getDevice(deviceID);
+        // Call extensions
+        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+    }
+
+    onMQTTMessage(payload) {
+        const {topic, message} = payload;
+        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
+
+        // Call extensions
+        this.callExtensionMethod('onMQTTMessage', [topic, message]);
+    }
+
+    async publishEntityState(IDorName, payload, stateChangeReason=null) {
+        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
+        if (!resolvedEntity || !resolvedEntity.settings) {
+            logger.error(`'${IDorName}' does not exist, skipping publish`);
+            return;
+        }
+
+        let messagePayload = {...payload};
+        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
+            this.state.get(resolvedEntity.settings.ID) : {};
+        const newState = objectAssignDeep.noMutate(currentState, payload);
+
+        // Update state cache with new state.
+        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
+
+        if (settings.get().advanced.cache_state) {
+            // Add cached state to payload
+            messagePayload = newState;
+        }
+
         const options = {
-            retain: deviceSettings.retain,
-            qos: deviceSettings.qos ? deviceSettings.qos : 0,
+            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
+            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
         };
 
-        this.mqtt.publish(deviceSettings.friendly_name, JSON.stringify(payload), options);
-    }
+        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention};
+        }
 
-    startupLogVersion(callback) {
-        const git = require('git-last-commit');
-        const packageJSON = require('../package.json');
-        const version = packageJSON.version;
+        const isDevice = resolvedEntity.type === 'device';
+        if (isDevice && settings.get().mqtt.include_device_information) {
+            const attributes = [
+                'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
+                'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
+            ];
 
-        git.getLastCommit((err, commit) => {
-            let commitHash = null;
+            messagePayload.device = {
+                friendlyName: resolvedEntity.name,
+                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
+            };
 
-            if (err) {
-                try {
-                    commitHash = require('../.hash.json').hash;
-                } catch (error) {
-                    commitHash = 'unknown';
-                }
-            } else {
-                commitHash = commit.shortHash;
+            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
+        }
+
+        // Add lastseen
+        const lastSeen = settings.get().advanced.last_seen;
+        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
+            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+        }
+
+        // Add device linkquality.
+        if (resolvedEntity.type === 'device' && resolvedEntity.device.linkquality !== undefined) {
+            messagePayload.linkquality = resolvedEntity.device.linkquality;
+        }
+
+        // filter mqtt message attributes
+        if (resolvedEntity.settings.filtered_attributes) {
+            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        }
+
+        if (Object.entries(messagePayload).length) {
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
             }
 
-            logger.info(`Starting zigbee2mqtt version ${version} (commit #${commitHash})`);
+            if (output === 'attribute_and_json' || output === 'attribute') {
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+            }
+        }
 
-            callback();
-        });
+        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload});
+    }
+
+    async iteratePayloadAttributeOutput(topicRoot, payload, options) {
+        for (const [key, value] of Object.entries(payload)) {
+            let subPayload = value;
+            let message = null;
+
+            // Special cases
+            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
+                subPayload = [subPayload.r, subPayload.g, subPayload.b];
+            }
+
+            // Check Array first, since it is also an Object
+            if (subPayload === null || subPayload === undefined) {
+                message = '';
+            } else if (Array.isArray(subPayload)) {
+                message = subPayload.map((x) => `${x}`).join(',');
+            } else if (typeof subPayload === 'object') {
+                this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
+            } else {
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
+            }
+
+            if (message !== null) {
+                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            }
+        }
+    }
+
+    async callExtensionMethod(method, parameters, extensions=null) {
+        for (const extension of extensions || this.extensions) {
+            if (extension[method]) {
+                try {
+                    await extension[method](...parameters);
+                } catch (error) {
+                    /* istanbul ignore next */
+                    logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
+                    /* istanbul ignore next */
+                    if (process.env.JEST_WORKER_ID !== undefined) {
+                        throw error;
+                    }
+                }
+            }
+        }
     }
 }
 

lib/eventBus.js

@@ -0,0 +1,43 @@
+const events = require('events');
+const assert = require('assert');
+
+const allowedEvents = [
+    'deviceRemoved', // Device has been removed
+    'deviceRenamed', // Device has been renamed
+    'groupRenamed', // Group has been renamed
+    'publishEntityState', // Entity state will be published
+    'stateChange', // Entity changes its state
+    'groupMembersChanged', // Members of a group has been changed
+    'reportingDisabled', // Reporting is disabled for a device
+    'devicesChanged', // Devices changed
+];
+
+class EventBus extends events.EventEmitter {
+    constructor() {
+        super();
+        this.callbackByExtension = {};
+    }
+
+    emit(event, data) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        super.emit(event, data);
+    }
+
+    on(event, callback, extension=null) {
+        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        if (extension) {
+            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
+            this.callbackByExtension[extension].push({event, callback});
+        }
+
+        super.on(event, callback);
+    }
+
+    removeListenersExtension(extension) {
+        for (const entry of this.callbackByExtension[extension] || []) {
+            super.removeListener(entry.event, entry.callback);
+        }
+    }
+}
+
+module.exports = EventBus;

lib/extension/availability.js

@@ -0,0 +1,227 @@
+const logger = require('../util/logger');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const Extension = require('./extension');
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availability`);
+
+// Pingable end devices, some end devices should be pinged
+// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
+const pingableEndDevices = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-N1EA'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
+];
+
+const Hours25 = 1000 * 60 * 60 * 25;
+const AvailabilityLagRatio = 0.1;
+
+
+function timeoutLag(timeout, ratio) {
+    const lag = timeout * ratio;
+    return Math.floor(Math.random() * Math.floor(lag));
+}
+
+/**
+ * This extensions pings devices to check if they are online.
+ */
+class Availability extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.availability_timeout = settings.get().advanced.availability_timeout;
+        this.timers = {};
+        this.state = {};
+
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+        this.eventBus.on('deviceRenamed', (data) => this.onDeviceRenamed(data), this.constructor.name);
+
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
+
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
+    }
+
+    onDeviceRenamed(data) {
+        this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+    }
+
+    onDeviceRemoved(resolvedEntity) {
+        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
+        delete this.state[resolvedEntity.device.ieeeAddr];
+        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
+    }
+
+    inPasslistOrNotInBlocklist(device) {
+        const ieeeAddr = device.ieeeAddr;
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings.friendlyName;
+
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
+        }
+
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
+            return false;
+        }
+
+        return true;
+    }
+
+    isPingable(device) {
+        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
+            return true;
+        }
+
+        // Device is a mains powered router
+        return device.type === 'Router' && device.powerSource !== 'Battery';
+    }
+
+    onMQTTConnected() {
+        for (const device of this.zigbee.getClients()) {
+            // Mark all devices as online on start
+            const ieeeAddr = device.ieeeAddr;
+            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
+
+            if (this.inPasslistOrNotInBlocklist(device)) {
+                if (this.isPingable(device)) {
+                    this.setTimerPingable(device);
+                } else {
+                    this.timers[ieeeAddr] = setInterval(() => {
+                        this.handleIntervalNotPingable(device);
+                    }, utils.secondsToMilliseconds(300));
+                }
+            }
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        // Clear topics for non-existing devices
+        const match = topic.match(topicRegex);
+        if (match && (!this.zigbee.resolveEntity(match[1]) || this.zigbee.resolveEntity(match[1]).name !== match[1])) {
+            this.mqtt.publish(`${match[1]}/availability`, null, {retain: true, qos: 0});
+        }
+    }
+
+    async handleIntervalPingable(device) {
+        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity) {
+            logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
+            return;
+        }
+
+        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
+        try {
+            await device.ping();
+            this.publishAvailability(device, true);
+            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
+        } catch (error) {
+            this.publishAvailability(device, false);
+            logger[level](`Failed to ping '${resolvedEntity.name}'`);
+        } finally {
+            this.setTimerPingable(device);
+        }
+    }
+
+    async handleIntervalNotPingable(device) {
+        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
+        if (!resolvedEntity || !device.lastSeen) {
+            return;
+        }
+
+        const ago = Date.now() - resolvedEntity.device.lastSeen;
+        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
+
+        if (ago > Hours25) {
+            this.publishAvailability(device, false);
+        }
+    }
+
+    setTimerPingable(device) {
+        const timeout = this.availability_timeout + timeoutLag(this.availability_timeout, AvailabilityLagRatio);
+        clearTimeout(this.timers[device.ieeeAddr]);
+        this.timers[device.ieeeAddr] = setTimeout(async () => {
+            await this.handleIntervalPingable(device);
+        }, utils.secondsToMilliseconds(timeout));
+    }
+
+    async stop() {
+        super.stop();
+        for (const timer of Object.values(this.timers)) {
+            clearTimeout(timer);
+        }
+
+        this.zigbee.getClients().forEach((device) => this.publishAvailability(device, false));
+    }
+
+    async onReconnect(device) {
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity && resolvedEntity.definition) {
+            try {
+                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
+                    if (converter) {
+                        await converter.convertGet(device.endpoints[0], key, {});
+                    }
+                }
+            } catch (error) {
+                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
+            }
+        }
+    }
+
+    publishAvailability(device, available, force=false) {
+        const ieeeAddr = device.ieeeAddr;
+        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
+            this.onReconnect(device);
+        }
+
+        const deviceSettings = settings.getDevice(ieeeAddr);
+        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
+        const topic = `${name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        if (this.state[ieeeAddr] !== available || force) {
+            this.state[ieeeAddr] = available;
+            this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+        if (!device) {
+            return;
+        }
+
+        if (this.inPasslistOrNotInBlocklist(device)) {
+            this.publishAvailability(data.device, true);
+
+            if (this.isPingable(device)) {
+                // When a zigbee message from a device is received we know the device is still alive.
+                // => reset the timer.
+                this.setTimerPingable(device);
+
+                const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
+                if (online && type === 'deviceAnnounce' && !utils.isIkeaTradfriDevice(device)) {
+                    /**
+                     * In case the device is powered off AND on within the availability timeout,
+                     * zigbee2qmtt does not detect the device as offline (device is still marked online).
+                     * When a device is turned on again the state could be out of sync.
+                     * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
+                     * deviceAnnounce is typically send when a device comes online.
+                     *
+                     * This isn't needed for TRADFRI devices as they already send the state themself.
+                     */
+                    this.onReconnect(device);
+                }
+            }
+        }
+    }
+}
+
+module.exports = Availability;

lib/extension/bind.js

@@ -0,0 +1,408 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+const debounce = require('debounce');
+const zigbeeHersdman = require('zigbee-herdsman');
+
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+
+// See zigbee-herdsman-converters devices.js
+const defaultBindGroup = {type: 'group_number', ID: 901};
+
+const defaultReportConfiguration = {
+    minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: 1,
+};
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') == null) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const reportClusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultReportConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultReportConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultReportConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultReportConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genLevelCtrl: [
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+                {type: 'commandToggle', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+];
+
+class Bind extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d));
+        this.pollDebouncers = {};
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let sourceKey = null;
+        let targetKey = null;
+        let clusters = null;
+
+        if (this.legacyApi && topic.match(legacyTopicRegex)) {
+            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0];
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = message;
+        } else if (topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
+            clusters = message.clusters;
+        }
+
+        return {type, sourceKey, targetKey, clusters};
+    }
+
+    async onMQTTMessage(topic, message) {
+        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
+        if (!type) return null;
+
+        let error = null;
+        const source = this.zigbee.resolveEntity(sourceKey);
+        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
+        const responseData = {from: sourceKey, to: targetKey};
+
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
+
+            let bindTarget = null;
+            if (target.type === 'group') bindTarget = target.group;
+            else if (target.type === 'group_number') bindTarget = target.ID;
+            else bindTarget = target.endpoint;
+
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusterCandidates) {
+                if (clusters && !clusters.includes(cluster)) continue;
+
+                const targetValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+
+                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
+
+                    try {
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    }
+                }
+            }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
+
+            if (successfulClusters.length !== 0) {
+                if (type === 'bind') {
+                    await this.setupReporting(source.endpoint.binds.filter((b) =>
+                        successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
+                } else if (target.type !== 'group_number') {
+                    await this.disableUnnecessaryReportings(bindTarget);
+                }
+            }
+        }
+
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit(`devicesChanged`);
+        }
+    }
+
+    async groupMembersChanged(data) {
+        const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
+            .flat().filter((b) => b.target === data.group.group);
+
+        if (data.action === 'add') {
+            await this.setupReporting(bindsToGroup);
+        } else { // action === remove/remove_all
+            await this.disableUnnecessaryReportings(data.endpoint);
+        }
+    }
+
+    getSetupReportingEndpoints(bind, coordinatorEp) {
+        const endpoints = bind.target.constructor.name === 'Group' ? bind.target.members : [bind.target];
+        return endpoints.filter((e) => {
+            const supportsInputCluster = e.supportsInputCluster(bind.cluster.name);
+            const hasConfiguredReporting = !!e.configuredReportings.find((c) => c.cluster.name === bind.cluster.name);
+            const hasBind = !!e.binds.find((b) => b.cluster.name === bind.cluster.name && b.target === coordinatorEp);
+            return supportsInputCluster && !(hasBind && hasConfiguredReporting);
+        });
+    }
+
+    async setupReporting(binds) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
+            for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
+                const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+                try {
+                    await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
+
+                    const items = await reportClusters[bind.cluster.name]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(bind.cluster.name, items);
+                    logger.info(`Succesfully setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                } catch (error) {
+                    logger.warn(`Failed to setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                }
+            }
+        }
+
+        this.eventBus.emit(`devicesChanged`);
+    }
+
+    async disableUnnecessaryReportings(target) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        const endpoints = target.constructor.name === 'Group' ? target.members : [target];
+        for (const endpoint of endpoints) {
+            const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+            const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
+                .map((b) => b.cluster.name);
+            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).flat().map((e) => e.binds)
+                .flat().filter((bind) => {
+                    if (bind.target.constructor.name === 'Group') {
+                        return bind.target.members.includes(endpoint);
+                    } else {
+                        return bind.target === endpoint;
+                    }
+                }).map((b) => b.cluster.name).filter((v, i, a) => a.indexOf(v) === i);
+
+            for (const cluster of boundClusters.filter((c) => !requiredClusters.includes(c) && c in reportClusters)) {
+                try {
+                    await endpoint.unbind(cluster, coordinatorEndpoint);
+                    const items = await reportClusters[cluster]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a, maximumReportInterval: 0xFFFF};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(cluster, items);
+                    logger.info(`Succesfully disabled reporting for '${entity}' cluster '${cluster}'`);
+                } catch (error) {
+                    logger.warn(`Failed to disable reporting for '${entity}' cluster '${cluster}'`);
+                }
+            }
+
+            this.eventBus.emit('reportingDisabled', {device: endpoint.getDevice()});
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
+                        continue;
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
+        }
+    }
+}
+
+module.exports = Bind;

lib/extension/bridge.js

@@ -0,0 +1,669 @@
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Transport = require('winston-transport');
+const stringify = require('json-stable-stringify-without-jsonify');
+const objectAssignDeep = require(`object-assign-deep`);
+const {updatedDiff, addedDiff} = require('deep-object-diff');
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+class Bridge extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
+        this.lastJoinedDeviceIeeeAddr = null;
+        this.setupMQTTLogging();
+        this.restartRequired = false;
+
+        this.requestLookup = {
+            'device/options': this.deviceOptions.bind(this),
+            'device/configure_reporting': this.deviceConfigureReporting.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/rename': this.deviceRename.bind(this),
+            'group/add': this.groupAdd.bind(this),
+            'group/options': this.groupOptions.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+            'group/rename': this.groupRename.bind(this),
+            'permit_join': this.permitJoin.bind(this),
+            'restart': this.restart.bind(this),
+            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'touchlink/identify': this.touchlinkIdentify.bind(this),
+            'touchlink/scan': this.touchlinkScan.bind(this),
+            'health_check': this.healthCheck.bind(this),
+            'options': this.bridgeOptions.bind(this),
+            // Below are deprecated
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/homeassistant': this.configHomeAssistant.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
+        };
+    }
+
+    async onMQTTConnected() {
+        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
+        this.eventBus.on(`devicesChanged`, () => {
+            this.publishDevices();
+            this.publishInfo();
+        });
+        this.eventBus.on(`deviceRenamed`, () => {
+            this.publishInfo();
+        });
+        this.eventBus.on(`groupRenamed`, () => {
+            this.publishInfo();
+        });
+        this.zigbee.on('permitJoinChanged', (data) => this.permitJoinChanged(data));
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+    }
+
+    setupMQTTLogging() {
+        const mqtt = this.mqtt;
+        class EventTransport extends Transport {
+            log(info, callback) {
+                const payload = stringify({message: info.message, level: info.level});
+                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
+                callback();
+            }
+        }
+
+        logger.addTransport(new EventTransport());
+    }
+
+    permitJoinChanged(data) {
+        if (!this.zigbee.isStopping()) {
+            this.publishInfo();
+        }
+    }
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
+        }
+
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') {
+                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
+            } else if (type === 'deviceInterview') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
+                };
+                if (data.status === 'successful') {
+                    const definition = resolvedEntity.definition;
+                    payload.supported = !!definition;
+                    payload.definition = this.getDefinitionPayload(definition);
+                }
+            } else if (type === 'deviceAnnounce') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
+                };
+            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
+
+            await this.mqtt.publish(
+                'bridge/event',
+                stringify({type: utils.toSnakeCase(type), data: payload}),
+                {retain: false, qos: 0},
+            );
+        }
+
+        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+            await this.publishDevices();
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    async deviceOptions(message) {
+        return this.changeEntityOptions('device', message);
+    }
+
+    async groupOptions(message) {
+        return this.changeEntityOptions('group', message);
+    }
+
+    async bridgeOptions(message) {
+        if (typeof message !== 'object' || typeof message.options !== 'object') {
+            throw new Error(`Invalid payload`);
+        }
+
+        const diffUpdated = updatedDiff(settings.get(), message.options);
+        const diffAdded = addedDiff(settings.get(), message.options);
+        const newSettings = objectAssignDeep.noMutate(diffUpdated, diffAdded);
+
+        // deep-object-diff converts arrays to objects, set original array back here
+        const convertBackArray = (before, after) => {
+            for (const [key, afterValue] of Object.entries(after)) {
+                const beforeValue = before[key];
+                if (Array.isArray(beforeValue)) {
+                    after[key] = beforeValue;
+                } else if (afterValue && typeof beforeValue === 'object') {
+                    convertBackArray(beforeValue, afterValue);
+                }
+            }
+        };
+        convertBackArray(message.options, newSettings);
+
+        const restartRequired = settings.apply(newSettings);
+        if (restartRequired) this.restartRequired = true;
+
+        // Apply some settings on-the-fly.
+        if (newSettings.hasOwnProperty('permit_join')) {
+            await this.zigbee.permitJoin(newSettings.permit_join);
+        }
+
+        if (newSettings.hasOwnProperty('homeassistant')) {
+            this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+        }
+
+        if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            logger.setLevel(newSettings.advanced.log_level);
+        }
+
+        logger.info('Succesfully changed options');
+        this.publishInfo();
+        return utils.getResponse(message, {restart_required: this.restartRequired}, null);
+    }
+
+    async deviceRemove(message) {
+        return this.removeEntity('device', message);
+    }
+
+    async groupRemove(message) {
+        return this.removeEntity('group', message);
+    }
+
+    async healthCheck(message) {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    async groupAdd(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    async deviceRename(message) {
+        return this.renameEntity('device', message);
+    }
+
+    async groupRename(message) {
+        return this.renameEntity('group', message);
+    }
+
+    async restart(message) {
+        // Wait 500 ms before restarting so response can be send.
+        setTimeout(this.restartCallback, 500);
+        logger.info('Restarting Zigbee2MQTT');
+        return utils.getResponse(message, {}, null);
+    }
+
+    async permitJoin(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value;
+        let time;
+        let resolvedEntity;
+        if (typeof message === 'object') {
+            value = message.value;
+            time = message.time;
+            if (message.device) {
+                resolvedEntity = this.zigbee.resolveEntity(message.device);
+                if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        if (typeof value === 'string') {
+            value = value.toLowerCase() === 'true';
+        }
+
+        await this.zigbee.permitJoin(value, resolvedEntity, time);
+        const response = {value};
+        if (resolvedEntity) response.device = message.device;
+        if (time) response.time = message.time;
+        return utils.getResponse(message, response, null);
+    }
+
+    // Deprecated
+    configLastSeen(message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configHomeAssistant(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        this.enableDisableExtension(value, 'HomeAssistant');
+        settings.set(['homeassistant'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configElapsed(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configLogLevel(message) {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    async touchlinkIdentify(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
+    async touchlinkFactoryReset(message) {
+        let result = false;
+        const payload = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, payload, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    async touchlinkScan(message) {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    getValue(message) {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
+
+            return message.value;
+        } else {
+            return message;
+        }
+    }
+
+    async changeEntityOptions(entityType, message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        settings.changeEntityOptions(ID, message.options);
+        const cleanup = (o) => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+        const oldOptions = cleanup(entity.settings);
+        const newOptions = cleanup(settings.getEntity(ID));
+        await this.publishInfo();
+
+        logger.info(`Changed config for ${entityType} ${ID}`);
+
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
+
+    async deviceConfigureReporting(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
+            !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
+            !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const ID = message.id;
+        const endpoint = this.getEntity('device', ID).endpoint;
+
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        await endpoint.bind(message.cluster, coordinatorEndpoint);
+
+        await endpoint.configureReporting(message.cluster, [{
+            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
+            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
+        }]);
+
+        this.publishDevices();
+
+        logger.info(`Configured reporting for '${message.id}', '${message.cluster}.${message.attribute}'`);
+
+        return utils.getResponse(message, {
+            id: message.id, cluster: message.cluster, maximum_report_interval: message.maximum_report_interval,
+            minimum_report_interval: message.minimum_report_interval, reportable_change: message.reportable_change,
+            attribute: message.attribute,
+        }, null);
+    }
+
+    renameEntity(entityType, message) {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
+        const entity = this.getEntity(entityType, from);
+
+        settings.changeFriendlyName(from, to);
+
+        // Clear retained messages
+        this.mqtt.publish(entity.name, '', {retain: true});
+
+        const oldFriendlyName = entity.settings.friendlyName;
+        if (entity.type === 'device') {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename, from: oldFriendlyName, to});
+        } else {
+            this.publishGroups();
+            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename, from: oldFriendlyName, to});
+        }
+
+        // Repulish entity state
+        this.publishEntityState(to, {});
+
+        return utils.getResponse(
+            message,
+            {from: oldFriendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
+    }
+
+    async removeEntity(entityType, message) {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity.type === 'device') {
+                if (block) {
+                    settings.blockDevice(entity.settings.ID);
+                }
+
+                if (force) {
+                    await entity.device.removeFromDatabase();
+                } else {
+                    await entity.device.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    await entity.group.removeFromDatabase();
+                } else {
+                    await entity.group.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+            }
+
+            // Remove from configuration.yaml
+            if (entity.type === 'device') {
+                settings.removeDevice(entity.settings.ID);
+            } else {
+                settings.removeGroup(entity.settings.ID);
+            }
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            // Clear any retained messages
+            this.mqtt.publish(entity.name, '', {retain: true});
+
+            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+
+            if (entity.type === 'device') {
+                this.publishGroups();
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type, ID) {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
+
+    async publishInfo() {
+        const config = objectAssignDeep.noMutate({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+            restart_required: this.restartRequired,
+            config,
+            config_schema: settings.schema,
+        };
+
+        await this.mqtt.publish(
+            'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    async publishDevices() {
+        const devices = this.zigbee.getDevices().map((device) => {
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const resolved = this.zigbee.resolveEntity(device);
+            const endpoints = {};
+            for (const endpoint of device.endpoints) {
+                const data = {
+                    bindings: [],
+                    configured_reportings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    let target;
+
+                    if (bind.target.constructor.name === 'Endpoint') {
+                        target = {
+                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
+                        };
+                    } else {
+                        target = {type: 'group', id: bind.target.groupID};
+                    }
+
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                for (const configuredReporting of endpoint.configuredReportings) {
+                    data.configured_reportings.push({
+                        cluster: configuredReporting.cluster.name,
+                        attribute: configuredReporting.attribute.name,
+                        minimum_report_interval: configuredReporting.minimumReportInterval,
+                        maximum_report_interval: configuredReporting.maximumReportInterval,
+                        reportable_change: configuredReporting.reportableChange,
+                    });
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
+
+            return {
+                ieee_address: device.ieeeAddr,
+                type: device.type,
+                network_address: device.networkAddress,
+                supported: !!definition,
+                friendly_name: resolved.name,
+                definition: this.getDefinitionPayload(definition),
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
+                model_id: device.modelID,
+                interviewing: device.interviewing,
+                interview_completed: device.interviewCompleted,
+                endpoints,
+            };
+        });
+
+        await this.mqtt.publish(
+            'bridge/devices', stringify(devices), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    async publishGroups() {
+        const groups = this.zigbee.getGroups().map((group) => {
+            const resolved = this.zigbee.resolveEntity(group);
+            return {
+                id: group.groupID,
+                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
+                members: group.members.map((m) => {
+                    return {
+                        ieee_address: m.deviceIeeeAddress,
+                        endpoint: m.ID,
+                    };
+                }),
+            };
+        });
+
+        await this.mqtt.publish(
+            'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    getDefinitionPayload(definition) {
+        if (definition) {
+            return {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                exposes: definition.exposes,
+                supports_ota: !!definition.ota,
+            };
+        } else {
+            return null;
+        }
+    }
+}
+
+module.exports = Bridge;

lib/extension/configure.js

@@ -0,0 +1,151 @@
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+class Configure extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.configuring = new Set();
+        this.onReportingDisabled = this.onReportingDisabled.bind(this);
+        this.attempts = {};
+
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
+    }
+
+    onReportingDisabled(data) {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        const device = data.device;
+
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity.device.meta && resolvedEntity.device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    shouldConfigure(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
+            return false;
+        }
+
+        if (resolvedEntity.device.meta &&
+            resolvedEntity.device.meta.hasOwnProperty('configured') &&
+            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            return false;
+        }
+
+        if (resolvedEntity.device.interviewing === true) {
+            return false;
+        }
+
+        return true;
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (topic === this.legacyTopic) {
+            const resolvedEntity = this.zigbee.resolveEntity(message);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                logger.error(`Device '${message}' does not exist`);
+                return;
+            }
+
+            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(resolvedEntity, true);
+        } else if (topic === this.topic) {
+            message = utils.parseJSON(message, message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const resolvedEntity = this.zigbee.resolveEntity(ID);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                error = `Device '${ID}' does not exist`;
+            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                error = `Device '${resolvedEntity.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(resolvedEntity, true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
+        }
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldConfigure(resolvedEntity)) {
+                await this.configure(resolvedEntity);
+            }
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        const device = data.device;
+
+        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity);
+        }
+    }
+
+    async configure(resolvedEntity, force=false, thowError=false) {
+        const device = resolvedEntity.device;
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return false;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${resolvedEntity.name}'`);
+        try {
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
+            logger.info(`Successfully configured '${resolvedEntity.name}'`);
+            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.save();
+            this.eventBus.emit(`devicesChanged`);
+        } catch (error) {
+            this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}
+
+module.exports = Configure;

lib/extension/extension.js

@@ -0,0 +1,54 @@
+class Extension {
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     */
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+    }
+
+    /**
+     * This method is called by the controller once Zigbee has been started.
+     */
+    // onZigbeeStarted() {}
+
+    /**
+     * This method is called by the controller once connected to the MQTT server.
+     */
+    // onMQTTConnected() {}
+
+    /**
+     * Is called when a Zigbee message from a device is received.
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} settingsDevice Device settings
+     */
+    // onZigbeeEvent(type, data, resolvedEntity) {}
+
+    /**
+     * Is called when a MQTT message is received
+     * @param {string} topic Topic on which the message was received
+     * @param {Object} message The received message
+     * @return {boolean} if the message was handled
+     */
+    // onMQTTMessage(topic, message) {}
+
+    /**
+     * Is called once the extension has to stop
+     */
+    stop() {
+        this.eventBus.removeListenersExtension(this.constructor.name);
+    }
+}
+
+module.exports = Extension;

lib/extension/externalConverters.js

@@ -0,0 +1,18 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+
+class ExternalConverters extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/externalExtension.js

@@ -0,0 +1,113 @@
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const fs = require('fs');
+const data = require('./../util/data');
+const path = require('path');
+const logger = require('./../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
+
+class ExternalExtension extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, addExtension, enableDisableExtension) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
+        this.addExtension = addExtension;
+        this.enableDisableExtension = enableDisableExtension;
+        this.extensionsBaseDir = 'extension';
+        this.loadExtension = this.loadExtension.bind(this);
+        this.requestLookup = {
+            'save': this.saveExtension.bind(this),
+            'remove': this.removeExtension.bind(this),
+        };
+        this.loadUserDefinedExtensions();
+    }
+    getExtensionsBasePath() {
+        return data.joinPath(this.extensionsBaseDir);
+    }
+
+    getListOfUserDefinedExtensions() {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
+                const extensonFilePath = path.join(basePath, fileName);
+                return {'name': fileName, 'code': fs.readFileSync(extensonFilePath, 'utf-8')};
+            });
+        } else {
+            return [];
+        }
+    }
+    removeExtension(message) {
+        const {name} = message;
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionToBeRemoved = extensions.find((e) => e.name === name);
+
+        if (extensionToBeRemoved) {
+            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            const basePath = this.getExtensionsBasePath();
+            const extensonFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensonFilePath);
+            this.publishExtensions();
+            logger.info(`Extension ${name} removed`);
+            return utils.getResponse(message, {}, null);
+        } else {
+            return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
+        }
+    }
+    saveExtension(message) {
+        const {name, code} = message;
+        const ModuleConstructor = utils.loadModuleFromText(code);
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, path.basename(name));
+        fs.writeFileSync(extensonFilePath, code);
+        this.publishExtensions();
+        logger.info(`Extension ${name} loaded`);
+        return utils.getResponse(message, {}, null);
+    }
+
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    loadExtension(ConstructorClass) {
+        this.enableDisableExtension(false, ConstructorClass.name);
+        this.addExtension(new ConstructorClass(...this.args, settings, logger));
+    }
+
+    loadUserDefinedExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        extensions
+            .map(({code}) => utils.loadModuleFromText(code))
+            .map(this.loadExtension);
+    }
+    async onMQTTConnected() {
+        this.publishExtensions();
+    }
+
+    async publishExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}
+
+module.exports = ExternalExtension;

lib/extension/frontend.js

@@ -0,0 +1,128 @@
+const http = require('http');
+const serveStatic = require('serve-static');
+const finalhandler = require('finalhandler');
+const Extension = require('./extension');
+const logger = require('../util/logger');
+const frontend = require('zigbee2mqtt-frontend');
+const WebSocket = require('ws');
+const url = require('url');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onRequest = this.onRequest.bind(this);
+        this.onUpgrade = this.onUpgrade.bind(this);
+        this.mqttBaseTopic = settings.get().mqtt.base_topic;
+        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
+        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
+        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+        this.host = settings.get().frontend.host || '0.0.0.0';
+        this.port = settings.get().frontend.port || 8080;
+        this.authToken = settings.get().frontend.auth_token || false;
+        this.retainedMessages = new Map();
+        /* istanbul ignore next */
+        const options = {setHeaders: (res, path) => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+    }
+
+    onZigbeeStarted() {
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    async stop() {
+        for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
+            client.close();
+        }
+
+        return new Promise((resolve) => {
+            this.server.close(resolve);
+        });
+    }
+    onRequest(request, response) {
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+    authenticate(request, cb) {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    onUpgrade(request, socket, head) {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    onWebSocketConnection(ws) {
+        ws.on('message', (message) => {
+            if (message) {
+                const {topic, payload} = utils.parseJSON(message, message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload = {};
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+            }
+
+            if (resolvedEntity.device.linkquality !== undefined) {
+                payload.linkquality = resolvedEntity.device.linkquality;
+            }
+
+            ws.send(stringify({topic: resolvedEntity.name, payload}));
+        }
+    }
+
+    onMQTTPublishedMessage(data) {
+        let {topic, payload, options} = data;
+        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            topic = topic.substring(this.mqttBaseTopic.length + 1);
+            payload = utils.parseJSON(payload, payload);
+            if (options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            for (const client of this.wss.clients) {
+                if (client.readyState === WebSocket.OPEN) {
+                    client.send(stringify({topic, payload}));
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -0,0 +1,330 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const postfixes = utils.getEndpointNames();
+const stringify = require('json-stable-stringify-without-jsonify');
+const equals = require('fast-deep-equal/es6');
+
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+
+class Groups extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onStateChange = this.onStateChange.bind(this);
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.lastOptimisticState = {};
+    }
+
+    async onZigbeeStarted() {
+        this.eventBus.on('stateChange', this.onStateChange);
+        await this.syncGroupsWithSettings();
+    }
+
+    async syncGroupsWithSettings() {
+        const settingsGroups = settings.getGroups();
+        const zigbeeGroups = this.zigbee.getGroups();
+
+        const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
+            try {
+                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
+                if (action === 'remove') {
+                    await endpoint.removeFromGroup(group);
+                } else {
+                    await endpoint.addToGroup(group);
+                }
+            } catch (error) {
+                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+            }
+        };
+
+        for (const settingGroup of settingsGroups) {
+            const groupID = settingGroup.ID;
+            const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
+            const settingsEntity = settingGroup.devices.map((d) => {
+                const entity = this.zigbee.resolveEntity(d);
+                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
+                return entity;
+            }).filter((e) => e != null);
+
+            // In settings but not in zigbee
+            for (const entity of settingsEntity) {
+                if (!zigbeeGroup.hasMember(entity.endpoint)) {
+                    addRemoveFromGroup('add', entity.name, settingGroup.friendlyName, entity.endpoint, zigbeeGroup);
+                }
+            }
+
+            // In zigbee but not in settings
+            for (const endpoint of zigbeeGroup.members) {
+                if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, settingGroup.friendlyName, endpoint, zigbeeGroup);
+                }
+            }
+        }
+
+        for (const zigbeeGroup of zigbeeGroups) {
+            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
+                for (const endpoint of zigbeeGroup.members) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.groupID, endpoint, zigbeeGroup);
+                }
+            }
+        }
+    }
+
+    async onStateChange(data) {
+        const reason = 'group_optimistic';
+        if (data.reason === reason) {
+            return;
+        }
+
+        const properties = ['state', 'brightness', 'color_temp', 'color'];
+        const payload = {};
+
+        properties.forEach((prop) => {
+            if (data.to.hasOwnProperty(prop)) {
+                payload[prop] = data.to[prop];
+            }
+        });
+
+        if (Object.keys(payload).length) {
+            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
+            const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
+                const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
+                return settingsGroup && settingsGroup.optimistic;
+            });
+
+            if (resolvedEntity.type === 'device') {
+                for (const zigbeeGroup of zigbeeGroups) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint) &&
+                        !equals(this.lastOptimisticState[zigbeeGroup.groupID], payload)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            this.lastOptimisticState[zigbeeGroup.groupID] = payload;
+                            await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
+                        }
+                    }
+                }
+            } else {
+                // Invalidate the last optimistic group state when group state is changed directly.
+                delete this.lastOptimisticState[resolvedEntity.group.groupID];
+
+                const groupIDsToPublish = new Set();
+                for (const member of resolvedEntity.group.members) {
+                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
+                    for (const zigbeeGroup of zigbeeGroups) {
+                        if (zigbeeGroup.hasMember(member)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                                groupIDsToPublish.add(zigbeeGroup.groupID);
+                            }
+                        }
+                    }
+                }
+                groupIDsToPublish.delete(resolvedEntity.group.groupID);
+                for (const groupID of groupIDsToPublish) {
+                    await this.publishEntityState(groupID, payload, reason);
+                }
+            }
+        }
+    }
+
+    areAllMembersOff(zigbeeGroup) {
+        for (const member of zigbeeGroup.members) {
+            const device = member.getDevice();
+            if (this.state.exists(device.ieeeAddr)) {
+                const state = this.state.get(device.ieeeAddr);
+                if (state && state.state === 'ON') {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    parseMQTTMessage(topic, message) {
+        let type = null;
+        let resolvedEntityGroup = null;
+        let resolvedEntityDevice = null;
+        let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
+
+        /* istanbul ignore else */
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
+
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return {};
+                }
+            } else {
+                type = 'remove_all';
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message);
+            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
+                logger.error(`Device '${message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return {};
+            }
+
+            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
+        }
+
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi,
+        };
+    }
+
+    async onMQTTMessage(topic, message) {
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey,
+        } = this.parseMQTTMessage(topic, message);
+        if (!type) return;
+
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
+        }
+
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
+
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                const endpointName = endpoints ?
+                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
+
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+                }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
+            }
+        }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit('groupMembersChanged',
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint});
+        }
+    }
+}
+
+module.exports = Groups;

lib/extension/homeassistant.js

@@ -0,0 +1,926 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const zigbee2mqttVersion = require('../../package.json').version;
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const assert = require('assert');
+
+const sensorClick = {
+    type: 'sensor',
+    object_id: 'click',
+    discovery_payload: {
+        icon: 'mdi:toggle-switch',
+        value_template: '{{ value_json.click }}',
+    },
+};
+
+const defaultStatusTopic = 'homeassistant/status';
+
+/**
+ * This extensions handles integration with HomeAssistant
+ */
+class HomeAssistant extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        // A map of all discoverd devices
+        this.discovered = {};
+        this.mapping = {};
+        this.discoveredTriggers = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
+
+        if (!settings.get().advanced.cache_state) {
+            logger.warn('In order for HomeAssistant integration to work properly set `cache_state: true');
+        }
+
+        if (settings.get().experimental.output === 'attribute') {
+            throw new Error('Home Assitant integration is not possible with attribute output!');
+        }
+
+        this.discoveryTopic = settings.get().advanced.homeassistant_discovery_topic;
+        this.statusTopic = settings.get().advanced.homeassistant_status_topic;
+
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data), this.constructor.name);
+        this.eventBus.on('deviceRenamed', (data) =>
+            this.onDeviceRenamed(data.device, data.homeAssisantRename), this.constructor.name,
+        );
+
+        this.populateMapping();
+    }
+
+    populateMapping() {
+        for (const def of zigbeeHerdsmanConverters.definitions) {
+            this.mapping[def.model] = [];
+
+            if (['WXKG01LM', 'HS1EB/HS1EB-E', 'ICZB-KPD14S', 'TERNCY-SD01', 'TERNCY-PP01', 'ICZB-KPD18S',
+                'E1766', 'ZWallRemote0', 'ptvo.switch', '2AJZ4KPKEY', 'ZGRC-KEY-013', 'HGZB-02S', 'HGZB-045',
+                'HGZB-1S', 'AV2010/34', 'IM6001-BTP01', 'WXKG11LM', 'WXKG03LM', 'WXKG02LM_rev1', 'WXKG02LM_rev2',
+                'QBKG04LM', 'QBKG03LM', 'QBKG11LM', 'QBKG21LM', 'QBKG22LM', 'WXKG12LM', 'QBKG12LM',
+                'E1743'].includes(def.model)) {
+                // deprecated
+                this.mapping[def.model].push(sensorClick);
+            }
+
+            if (['ICTC-G-1'].includes(def.model)) {
+                // deprecated
+                this.mapping[def.model].push({
+                    type: 'sensor',
+                    object_id: 'brightness',
+                    discovery_payload: {
+                        unit_of_measurement: 'brightness',
+                        icon: 'mdi:brightness-5',
+                        value_template: '{{ value_json.brightness }}',
+                    },
+                });
+            }
+
+            for (const expose of def.exposes) {
+                let discoveryEntry = null;
+                /* istanbul ignore else */
+                if (expose.type === 'light') {
+                    const supportsXY = !!expose.features.find((e) => e.name === 'color_xy');
+                    const supportsHS = !!expose.features.find((e) => e.name === 'color_hs');
+                    const colorTemp = expose.features.find((e) => e.name === 'color_temp');
+                    discoveryEntry = {
+                        type: 'light',
+                        object_id: expose.endpoint ? `light_${expose.endpoint}` : 'light',
+                        discovery_payload: {
+                            brightness: !!expose.features.find((e) => e.name === 'brightness'),
+                            color_temp: !!colorTemp,
+                            xy: supportsXY,
+                            hs: !supportsXY && supportsHS,
+                            schema: 'json',
+                            command_topic: true,
+                            brightness_scale: 254,
+                            command_topic_prefix: expose.endpoint ? expose.endpoint : undefined,
+                            state_topic_postfix: expose.endpoint ? expose.endpoint : undefined,
+                        },
+                    };
+
+                    if (colorTemp) {
+                        discoveryEntry.discovery_payload.max_mireds = colorTemp.value_max;
+                        discoveryEntry.discovery_payload.min_mireds = colorTemp.value_min;
+                    }
+
+                    const effect = def.exposes.find((e) => e.type === 'enum' && e.name === 'effect');
+                    if (effect) {
+                        discoveryEntry.discovery_payload.effect = true;
+                        discoveryEntry.discovery_payload.effect_list = effect.values;
+                    }
+                } else if (expose.type === 'switch') {
+                    const state = expose.features.find((f) => f.name === 'state');
+                    discoveryEntry = {
+                        type: 'switch',
+                        object_id: expose.endpoint ? `switch_${expose.endpoint}` : 'switch',
+                        discovery_payload: {
+                            payload_off: state.value_off,
+                            payload_on: state.value_on,
+                            value_template: `{{ value_json.${state.property} }}`,
+                            command_topic: true,
+                            command_topic_prefix: expose.endpoint ? expose.endpoint : undefined,
+                        },
+                    };
+
+                    const different = ['valve_detection', 'window_detection', 'auto_lock', 'away_mode'];
+                    if (different.includes(state.property)) {
+                        discoveryEntry.discovery_payload.command_topic_postfix = state.property;
+                        discoveryEntry.discovery_payload.state_off = state.value_off;
+                        discoveryEntry.discovery_payload.state_on = state.value_on;
+                        discoveryEntry.discovery_payload.state_topic = true;
+                        discoveryEntry.object_id = state.property;
+
+                        if (state.property === 'window_detection') {
+                            discoveryEntry.discovery_payload.icon = 'mdi:window-open-variant';
+                        }
+                    }
+                } else if (expose.type === 'climate') {
+                    const setpointProperties = ['occupied_heating_setpoint', 'current_heating_setpoint'];
+                    const setpoint = expose.features.find((f) => setpointProperties.includes(f.name));
+                    assert(setpoint, 'No setpoint found');
+                    const temperature = expose.features.find((f) => f.name === 'local_temperature');
+                    assert(temperature, 'No temperature found');
+
+                    discoveryEntry = {
+                        type: 'climate',
+                        object_id: expose.endpoint ? `climate_${expose.endpoint}` : 'climate',
+                        discovery_payload: {
+                            // Static
+                            state_topic: false,
+                            temperature_unit: 'C',
+                            // Setpoint
+                            temp_step: setpoint.value_step,
+                            min_temp: setpoint.value_min.toString(),
+                            max_temp: setpoint.value_max.toString(),
+                            // Temperature
+                            current_temperature_topic: true,
+                            current_temperature_template: `{{ value_json.${temperature.property} }}`,
+                        },
+                    };
+
+                    const mode = expose.features.find((f) => f.name === 'system_mode');
+                    if (mode) {
+                        discoveryEntry.discovery_payload.mode_state_topic = true;
+                        discoveryEntry.discovery_payload.mode_state_template = `{{ value_json.${mode.property} }}`;
+                        discoveryEntry.discovery_payload.modes = mode.values;
+                        discoveryEntry.discovery_payload.mode_command_topic = true;
+                    }
+
+                    const state = expose.features.find((f) => f.name === 'running_state');
+                    if (state) {
+                        discoveryEntry.discovery_payload.action_topic = true;
+                        discoveryEntry.discovery_payload.action_template = `{% set values = ` +
+                                `{'idle':'off','heat':'heating','cool':'cooling','fan only':'fan'}` +
+                                ` %}{{ values[value_json.${state.property}] }}`;
+                    }
+
+                    const coolingSetpoint = expose.features.find((f) => f.name === 'occupied_cooling_setpoint');
+                    if (coolingSetpoint) {
+                        discoveryEntry.discovery_payload.temperature_low_command_topic = setpoint.name;
+                        discoveryEntry.discovery_payload.temperature_low_state_template =
+                            `{{ value_json.${setpoint.property} }}`;
+                        discoveryEntry.discovery_payload.temperature_low_state_topic = true;
+                        discoveryEntry.discovery_payload.temperature_high_command_topic = coolingSetpoint.name;
+                        discoveryEntry.discovery_payload.temperature_high_state_template =
+                            `{{ value_json.${coolingSetpoint.property} }}`;
+                        discoveryEntry.discovery_payload.temperature_high_state_topic = true;
+                    } else {
+                        discoveryEntry.discovery_payload.temperature_command_topic = setpoint.name;
+                        discoveryEntry.discovery_payload.temperature_state_template =
+                            `{{ value_json.${setpoint.property} }}`;
+                        discoveryEntry.discovery_payload.temperature_state_topic = true;
+                    }
+
+                    const fanMode = expose.features.find((f) => f.name === 'fan_mode');
+                    if (fanMode) {
+                        discoveryEntry.discovery_payload.fan_modes = fanMode.values;
+                        discoveryEntry.discovery_payload.fan_mode_command_topic = true;
+                        discoveryEntry.discovery_payload.fan_mode_state_template =
+                            `{{ value_json.${fanMode.property} }}`;
+                        discoveryEntry.discovery_payload.fan_mode_state_topic = true;
+                    }
+
+                    const preset = expose.features.find((f) => f.name === 'preset');
+                    if (preset) {
+                        discoveryEntry.discovery_payload.hold_modes = preset.values;
+                        discoveryEntry.discovery_payload.hold_command_topic = true;
+                        discoveryEntry.discovery_payload.hold_state_template =
+                            `{{ value_json.${preset.property} }}`;
+                        discoveryEntry.discovery_payload.hold_state_topic = true;
+                    }
+
+                    const awayMode = expose.features.find((f) => f.name === 'away_mode');
+                    if (awayMode) {
+                        discoveryEntry.discovery_payload.away_mode_command_topic = true;
+                        discoveryEntry.discovery_payload.away_mode_state_topic = true;
+                        discoveryEntry.discovery_payload.away_mode_state_template =
+                            `{{ value_json.${awayMode.property} }}`;
+                    }
+
+                    if (expose.endpoint) {
+                        discoveryEntry.discovery_payload.state_topic_postfix = expose.endpoint;
+                    }
+                } else if (expose.type === 'lock') {
+                    assert(!expose.endpoint, `Endpoint not supported for lock type`);
+                    const state = expose.features.find((f) => f.name === 'state');
+                    assert(state, 'No state found');
+                    discoveryEntry = {
+                        type: 'lock',
+                        object_id: 'lock',
+                        discovery_payload: {
+                            command_topic: true,
+                            value_template: `{{ value_json.${state.property} }}`,
+                        },
+                    };
+
+                    if (state.property === 'keypad_lockout') {
+                        // deprecated: keypad_lockout is messy, but changing is breaking
+                        discoveryEntry.discovery_payload.payload_lock = state.value_on;
+                        discoveryEntry.discovery_payload.payload_unlock = state.value_off;
+                        discoveryEntry.discovery_payload.state_topic = true;
+                        discoveryEntry.object_id = 'keypad_lock';
+                    } else if (state.property === 'child_lock') {
+                        // deprecated: child_lock is messy, but changing is breaking
+                        discoveryEntry.discovery_payload.payload_lock = state.value_on;
+                        discoveryEntry.discovery_payload.payload_unlock = state.value_off;
+                        discoveryEntry.discovery_payload.state_locked = 'LOCKED';
+                        discoveryEntry.discovery_payload.state_unlocked = 'UNLOCKED';
+                        discoveryEntry.discovery_payload.state_topic = true;
+                        discoveryEntry.object_id = 'child_lock';
+                    } else {
+                        discoveryEntry.discovery_payload.state_locked = state.value_on;
+                        discoveryEntry.discovery_payload.state_unlocked = state.value_off;
+                    }
+
+                    if (state.property !== 'state') {
+                        discoveryEntry.discovery_payload.command_topic_postfix = state.property;
+                    }
+                } else if (expose.type === 'cover') {
+                    assert(!expose.endpoint, `Endpoint not supported for cover type`);
+                    const hasPosition = expose.features.find((e) => e.name === 'position');
+                    const hasTilt = expose.features.find((e) => e.name === 'tilt');
+
+                    discoveryEntry = {
+                        type: 'cover',
+                        object_id: 'cover',
+                        discovery_payload: {
+                            command_topic: true,
+                            state_topic: !hasPosition,
+                        },
+                    };
+
+                    if (!hasPosition && !hasTilt) {
+                        discoveryEntry.discovery_payload.optimistic = true;
+                    }
+
+                    if (hasPosition) {
+                        discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
+                            value_template: '{{ value_json.position }}',
+                            set_position_template: '{ "position": {{ position }} }',
+                            set_position_topic: true,
+                            position_topic: true,
+                        };
+                    }
+
+                    if (hasTilt) {
+                        discoveryEntry.discovery_payload = {...discoveryEntry.discovery_payload,
+                            tilt_command_topic: true,
+                            tilt_status_topic: true,
+                            tilt_status_template: '{{ value_json.tilt }}',
+                        };
+                    }
+                } else if (expose.type === 'fan') {
+                    assert(!expose.endpoint, `Endpoint not supported for fan type`);
+                    discoveryEntry = {
+                        type: 'fan',
+                        object_id: 'fan',
+                        discovery_payload: {
+                            state_topic: true,
+                            state_value_template: '{{ value_json.fan_state }}',
+                            command_topic: true,
+                            command_topic_postfix: 'fan_state',
+                        },
+                    };
+
+                    const speed = expose.features.find((e) => e.name === 'mode');
+                    if (speed) {
+                        discoveryEntry.discovery_payload.speed_state_topic = true;
+                        discoveryEntry.discovery_payload.speed_command_topic = true;
+                        discoveryEntry.discovery_payload.speed_value_template = '{{ value_json.fan_mode }}';
+                        discoveryEntry.discovery_payload.speeds = speed.values;
+                    }
+                } else if (expose.type === 'binary') {
+                    const lookup = {
+                        occupancy: {device_class: 'motion'},
+                        battery_low: {device_class: 'battery'},
+                        water_leak: {device_class: 'moisture'},
+                        vibration: {device_class: 'vibration'},
+                        contact: {device_class: 'door'},
+                        smoke: {device_class: 'smoke'},
+                        gas: {device_class: 'gas'},
+                        carbon_monoxide: {device_class: 'safety'},
+                        presence: {device_class: 'presence'},
+                    };
+
+                    discoveryEntry = {
+                        type: 'binary_sensor',
+                        object_id: expose.endpoint ? `${expose.name}_${expose.endpoint}` : `${expose.name}`,
+                        discovery_payload: {
+                            value_template: `{{ value_json.${expose.property} }}`,
+                            payload_on: expose.value_on,
+                            payload_off: expose.value_off,
+                            ...(lookup[expose.name] || {}),
+                        },
+                    };
+                } else if (expose.type === 'numeric') {
+                    const lookup = {
+                        battery: {device_class: 'battery'},
+                        temperature: {device_class: 'temperature'},
+                        humidity: {device_class: 'humidity'},
+                        illuminance_lux: {device_class: 'illuminance'},
+                        illuminance: {device_class: 'illuminance'},
+                        soil_moisture: {icon: 'mdi:water-percent'},
+                        position: {icon: 'mdi:valve'},
+                        pressure: {device_class: 'pressure'},
+                        power: {device_class: 'power'},
+                        linkquality: {icon: 'mdi:signal'},
+                        current: {device_class: 'current'},
+                        voltage: {device_class: 'voltage'},
+                        current_phase_b: {device_class: 'current'},
+                        voltage_phase_b: {device_class: 'voltage'},
+                        current_phase_c: {device_class: 'current'},
+                        voltage_phase_c: {device_class: 'voltage'},
+                        energy: {device_class: 'energy'},
+                        smoke_density: {icon: 'mdi:google-circles-communities'},
+                        gas_density: {icon: 'mdi:google-circles-communities'},
+                        pm25: {icon: 'mdi:air-filter'},
+                        pm10: {icon: 'mdi:air-filter'},
+                        voc: {icon: 'mdi:air-filter'},
+                        aqi: {icon: 'mdi:air-filter'},
+                        hcho: {icon: 'mdi:air-filter'},
+                        requested_brightness_level: {icon: 'mdi:brightness-5'},
+                        requested_brightness_percent: {icon: 'mdi:brightness-5'},
+                        eco2: {icon: 'mdi:molecule-co2'},
+                        co2: {icon: 'mdi:molecule-co2'},
+                        local_temperature: {device_class: 'temperature'},
+                        x_axis: {icon: 'mdi:axis-x-arrow'},
+                        y_axis: {icon: 'mdi:axis-y-arrow'},
+                        z_axis: {icon: 'mdi:axis-z-arrow'},
+                    };
+
+                    discoveryEntry = {
+                        type: 'sensor',
+                        object_id: expose.endpoint ? `${expose.name}_${expose.endpoint}` : `${expose.name}`,
+                        discovery_payload: {
+                            value_template: `{{ value_json.${expose.property} }}`,
+                            ...(expose.unit && {unit_of_measurement: expose.unit}),
+                            ...lookup[expose.name],
+                        },
+                    };
+                } else if (expose.type === 'enum' || expose.type === 'text' || expose.type === 'composite') {
+                    const ACCESS_STATE = 1;
+                    if (expose.access & ACCESS_STATE) {
+                        const lookup = {
+                            action: {icon: 'mdi:gesture-double-tap'},
+                        };
+
+                        discoveryEntry = {
+                            type: 'sensor',
+                            object_id: expose.property,
+                            discovery_payload: {
+                                value_template: `{{ value_json.${expose.property} }}`,
+                                ...lookup[expose.name],
+                            },
+                        };
+                    }
+                } else {
+                    throw new Error(`Unsupported exposes type: '${expose.type}'`);
+                }
+
+                if (discoveryEntry) {
+                    this.mapping[def.model].push(discoveryEntry);
+                }
+            }
+        }
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            if (definition.hasOwnProperty('homeassistant')) {
+                this.mapping[definition.model] = definition.homeassistant;
+            }
+        }
+    }
+
+    onDeviceRemoved(resolvedEntity) {
+        logger.debug(`Clearing Home Assistant discovery topic for '${resolvedEntity.name}'`);
+        delete this.discovered[resolvedEntity.device.ieeeAddr];
+        for (const config of this.getConfigs(resolvedEntity)) {
+            const topic = this.getDiscoveryTopic(config, resolvedEntity.device);
+            this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
+        }
+    }
+
+    async onPublishEntityState(data) {
+        /**
+         * In case we deal with a lightEndpoint configuration Zigbee2MQTT publishes
+         * e.g. {state_l1: ON, brightness_l1: 250} to zigbee2mqtt/mydevice.
+         * As the Home Assistant MQTT JSON light cannot be configured to use state_l1/brightness_l1
+         * as the state variables, the state topic is set to zigbee2mqtt/mydevice/l1.
+         * Here we retrieve all the attributes with the _l1 values and republish them on
+         * zigbee2mqtt/mydevice/l1.
+         */
+        if (data.entity.definition && this.mapping[data.entity.definition.model]) {
+            for (const config of this.mapping[data.entity.definition.model]) {
+                const match = /light_(.*)/.exec(config['object_id']);
+                if (match) {
+                    const endpoint = match[1];
+                    const endpointRegExp = new RegExp(`(.*)_${endpoint}`);
+                    const payload = {};
+                    for (const key of Object.keys(data.messagePayload)) {
+                        const keyMatch = endpointRegExp.exec(key);
+                        if (keyMatch) {
+                            payload[keyMatch[1]] = data.messagePayload[key];
+                        }
+                    }
+
+                    await this.mqtt.publish(
+                        `${data.entity.name}/${endpoint}`, stringify(payload), {},
+                    );
+                }
+            }
+        }
+
+        /**
+         * Publish an empty value for click and action payload, in this way Home Assistant
+         * can use Home Assistant entities in automations.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/959#issuecomment-480341347
+         */
+        if (settings.get().advanced.homeassistant_legacy_triggers) {
+            const keys = ['action', 'click'].filter((k) =>
+                data.messagePayload.hasOwnProperty(k) && data.messagePayload[k] !== '');
+            for (const key of keys) {
+                this.publishEntityState(data.entity.device.ieeeAddr, {[key]: ''});
+            }
+        }
+
+        /**
+         * Implements the MQTT device trigger (https://www.home-assistant.io/integrations/device_trigger.mqtt/)
+         * The MQTT device trigger does not support JSON parsing, so it cannot listen to zigbee2mqtt/my_device
+         * Whenever a device publish an {action: *} we discover an MQTT device trigger sensor
+         * and republish it to zigbee2mqtt/my_devic/action
+         */
+        if (data.entity.definition) {
+            const keys = ['action', 'click'].filter((k) => data.messagePayload[k] && data.messagePayload[k] !== '');
+            for (const key of keys) {
+                const value = data.messagePayload[key].toString();
+                await this.publishDeviceTriggerDiscover(data.entity, key, value);
+                await this.mqtt.publish(`${data.entity.name}/${key}`, value, {});
+            }
+        }
+
+        /**
+         * Publish a value for update_available (if not there yet) to prevent Home Assistant generating warnings of
+         * this value not being available.
+         */
+        const supportsOTA = data.entity.definition && data.entity.definition.hasOwnProperty('ota');
+        const mockedValues = [
+            {
+                property: 'update_available',
+                condition: supportsOTA && this.legacyApi,
+                value: false,
+            },
+            {
+                property: 'update',
+                condition: supportsOTA,
+                value: {state: 'idle'},
+            },
+            {
+                property: 'water_leak',
+                condition: data.entity.device && data.entity.definition && this.mapping[data.entity.definition.model] &&
+                    this.mapping[data.entity.definition.model].filter((c) => c.object_id === 'water_leak').length === 1,
+                value: false,
+            },
+        ];
+
+        for (const entry of mockedValues) {
+            if (entry.condition && !data.messagePayload.hasOwnProperty(entry.property)) {
+                logger.debug(`Mocking '${entry.property}' value for Home Assistant`);
+                this.publishEntityState(data.entity.device.ieeeAddr, {[entry.property]: entry.value});
+            }
+        }
+    }
+
+    onDeviceRenamed(device, homeAssisantRename) {
+        logger.debug(`Refreshing Home Assistant discovery topic for '${device.ieeeAddr}'`);
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+
+        // Clear before rename so Home Assistant uses new friendly_name
+        // https://github.com/Koenkk/zigbee2mqtt/issues/4096#issuecomment-674044916
+        if (homeAssisantRename) {
+            for (const config of this.getConfigs(resolvedEntity)) {
+                const topic = this.getDiscoveryTopic(config, device);
+                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
+            }
+        }
+
+        this.discover(resolvedEntity, true);
+
+        if (this.discoveredTriggers[device.ieeeAddr]) {
+            for (const config of this.discoveredTriggers[device.ieeeAddr]) {
+                const key = config.substring(0, config.indexOf('_'));
+                const value = config.substring(config.indexOf('_') + 1);
+                this.publishDeviceTriggerDiscover(resolvedEntity, key, value, true);
+            }
+        }
+    }
+
+    async onMQTTConnected() {
+        this.mqtt.subscribe(this.statusTopic);
+        this.mqtt.subscribe(defaultStatusTopic);
+        this.mqtt.subscribe(`${this.discoveryTopic}/#`);
+
+        // MQTT discovery of all paired devices on startup.
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.discover(resolvedEntity, true);
+        }
+    }
+
+    getConfigs(resolvedEntity) {
+        if (!resolvedEntity || !resolvedEntity.definition || !this.mapping[resolvedEntity.definition.model]) return [];
+
+        let configs = this.mapping[resolvedEntity.definition.model].slice();
+        if (resolvedEntity.definition.hasOwnProperty('ota')) {
+            const updateStateSensor = {
+                type: 'sensor',
+                object_id: 'update_state',
+                discovery_payload: {
+                    icon: 'mdi:update',
+                    value_template: `{{ value_json['update']['state'] }}`,
+                },
+            };
+
+            configs.push(updateStateSensor);
+            if (this.legacyApi) {
+                const updateAvailableSensor = {
+                    type: 'binary_sensor',
+                    object_id: 'update_available',
+                    discovery_payload: {
+                        payload_on: true,
+                        payload_off: false,
+                        value_template: '{{ value_json.update_available}}',
+                    },
+                };
+                configs.push(updateAvailableSensor);
+            }
+        }
+
+        if (resolvedEntity.settings.hasOwnProperty('legacy') && !resolvedEntity.settings.legacy) {
+            configs = configs.filter((c) => c !== sensorClick);
+        }
+
+        if (!settings.get().advanced.homeassistant_legacy_triggers) {
+            configs = configs.filter((c) => c.object_id !== 'action' && c.object_id !== 'click');
+        }
+
+        // deep clone of the config objects
+        configs = JSON.parse(JSON.stringify(configs));
+
+        if (resolvedEntity.settings.homeassistant) {
+            const s = resolvedEntity.settings.homeassistant;
+            configs = configs.filter((config) => !s.hasOwnProperty(config.object_id) || s[config.object_id] != null);
+            configs.forEach((config) => {
+                const configOverride = s[config.object_id];
+                if (configOverride) {
+                    config.object_id = configOverride.object_id || config.object_id;
+                    config.type = configOverride.type || config.type;
+                }
+            });
+        }
+
+        return configs;
+    }
+
+    discover(resolvedEntity, force=false) {
+        // Check if already discoverd and check if there are configs.
+        const {device, definition} = resolvedEntity;
+        const discover = force || !this.discovered[device.ieeeAddr];
+        if (!discover || !device || !definition || !this.mapping[definition.model] || device.interviewing ||
+            (resolvedEntity.settings.hasOwnProperty('homeassistant') && !resolvedEntity.settings.homeassistant)) {
+            return;
+        }
+
+        const friendlyName = resolvedEntity.settings.friendlyName;
+        this.getConfigs(resolvedEntity).forEach((config) => {
+            const payload = {...config.discovery_payload};
+            let stateTopic = `${settings.get().mqtt.base_topic}/${friendlyName}`;
+            if (payload.state_topic_postfix) {
+                stateTopic += `/${payload.state_topic_postfix}`;
+                delete payload.state_topic_postfix;
+            }
+
+            if (!payload.hasOwnProperty('state_topic') || payload.state_topic) {
+                payload.state_topic = stateTopic;
+            } else {
+                /* istanbul ignore else */
+                if (payload.hasOwnProperty('state_topic')) {
+                    delete payload.state_topic;
+                }
+            }
+
+            if (payload.position_topic) {
+                payload.position_topic = stateTopic;
+            }
+
+            if (payload.tilt_status_topic) {
+                payload.tilt_status_topic = stateTopic;
+            }
+
+            payload.json_attributes_topic = stateTopic;
+
+            // Set (unique) name, separate by space if friendlyName contains space.
+            const nameSeparator = friendlyName.includes('_') ? '_' : ' ';
+            payload.name = friendlyName;
+            if (config.object_id.startsWith(config.type) && config.object_id.includes('_')) {
+                payload.name += `${nameSeparator}${config.object_id.split(/_(.+)/)[1]}`;
+            } else if (!config.object_id.startsWith(config.type)) {
+                payload.name += `${nameSeparator}${config.object_id.replace(/_/g, nameSeparator)}`;
+            }
+
+            // Set unique_id
+            payload.unique_id = `${resolvedEntity.settings.ID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
+
+            // Attributes for device registry
+            payload.device = this.getDevicePayload(resolvedEntity);
+
+            // Availability payload
+            payload.availability = [{topic: `${settings.get().mqtt.base_topic}/bridge/state`}];
+            if (settings.get().advanced.availability_timeout) {
+                payload.availability.push({topic: `${settings.get().mqtt.base_topic}/${friendlyName}/availability`});
+            }
+
+            if (payload.command_topic) {
+                payload.command_topic = `${settings.get().mqtt.base_topic}/${friendlyName}/`;
+
+                if (payload.command_topic_prefix) {
+                    payload.command_topic += `${payload.command_topic_prefix}/`;
+                    delete payload.command_topic_prefix;
+                }
+
+                payload.command_topic += 'set';
+
+                if (payload.command_topic_postfix) {
+                    payload.command_topic += `/${payload.command_topic_postfix}`;
+                    delete payload.command_topic_postfix;
+                }
+            }
+
+            if (payload.set_position_topic && payload.command_topic) {
+                payload.set_position_topic = payload.command_topic;
+            }
+
+            if (payload.tilt_command_topic && payload.command_topic) {
+                // Home Assistant does not support templates to set tilt (as of 2019-08-17),
+                // so we (have to) use a subtopic.
+                payload.tilt_command_topic = payload.command_topic + '/tilt';
+            }
+
+            if (payload.mode_state_topic) {
+                payload.mode_state_topic = stateTopic;
+            }
+
+            if (payload.mode_command_topic) {
+                payload.mode_command_topic = `${stateTopic}/set/system_mode`;
+            }
+
+            if (payload.hold_command_topic) {
+                payload.hold_command_topic = `${stateTopic}/set/preset`;
+            }
+
+            if (payload.hold_state_topic) {
+                payload.hold_state_topic = stateTopic;
+            }
+
+            if (payload.away_mode_state_topic) {
+                payload.away_mode_state_topic = stateTopic;
+            }
+
+            if (payload.away_mode_command_topic) {
+                payload.away_mode_command_topic = `${stateTopic}/set/away_mode`;
+            }
+
+            if (payload.current_temperature_topic) {
+                payload.current_temperature_topic = stateTopic;
+            }
+
+            if (payload.temperature_state_topic) {
+                payload.temperature_state_topic = stateTopic;
+            }
+
+            if (payload.temperature_low_state_topic) {
+                payload.temperature_low_state_topic = stateTopic;
+            }
+
+            if (payload.temperature_high_state_topic) {
+                payload.temperature_high_state_topic = stateTopic;
+            }
+
+            if (payload.speed_state_topic) {
+                payload.speed_state_topic = stateTopic;
+            }
+
+            if (payload.temperature_command_topic) {
+                payload.temperature_command_topic = `${stateTopic}/set/${payload.temperature_command_topic}`;
+            }
+
+            if (payload.temperature_low_command_topic) {
+                payload.temperature_low_command_topic = `${stateTopic}/set/${payload.temperature_low_command_topic}`;
+            }
+
+            if (payload.temperature_high_command_topic) {
+                payload.temperature_high_command_topic = `${stateTopic}/set/${payload.temperature_high_command_topic}`;
+            }
+
+            if (payload.fan_mode_state_topic) {
+                payload.fan_mode_state_topic = stateTopic;
+            }
+
+            if (payload.fan_mode_command_topic) {
+                payload.fan_mode_command_topic = `${stateTopic}/set/fan_mode`;
+            }
+
+            if (payload.speed_command_topic) {
+                payload.speed_command_topic = `${stateTopic}/set/fan_mode`;
+            }
+
+            if (payload.action_topic) {
+                payload.action_topic = stateTopic;
+            }
+
+            // Override configuration with user settings.
+            if (resolvedEntity.settings.hasOwnProperty('homeassistant')) {
+                const add = (obj) => {
+                    Object.keys(obj).forEach((key) => {
+                        if (['type', 'object_id'].includes(key)) {
+                            return;
+                        } else if (['number', 'string', 'boolean'].includes(typeof obj[key])) {
+                            payload[key] = obj[key];
+                        } else if (obj[key] === null) {
+                            delete payload[key];
+                        } else if (key === 'device' && typeof obj[key] === 'object') {
+                            Object.keys(obj['device']).forEach((key) => {
+                                payload['device'][key] = obj['device'][key];
+                            });
+                        }
+                    });
+                };
+
+                add(resolvedEntity.settings.homeassistant);
+
+                if (resolvedEntity.settings.homeassistant.hasOwnProperty(config.object_id)) {
+                    add(resolvedEntity.settings.homeassistant[config.object_id]);
+                }
+            }
+
+            const topic = this.getDiscoveryTopic(config, device);
+            this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
+        });
+
+        this.discovered[device.ieeeAddr] = true;
+    }
+
+    onMQTTMessage(topic, message) {
+        const discoveryRegex = new RegExp(`${this.discoveryTopic}/(.*)/(.*)/(.*)/config`);
+        const discoveryMatch = topic.match(discoveryRegex);
+        const isDeviceAutomation = discoveryMatch && discoveryMatch[1] === 'device_automation';
+        if (discoveryMatch) {
+            // Clear outdated discovery configs and remember already discoverd device_automations
+            try {
+                message = JSON.parse(message);
+                const baseTopic = settings.get().mqtt.base_topic + '/';
+                if (isDeviceAutomation && (!message.topic || !message.topic.startsWith(baseTopic))) {
+                    return;
+                }
+
+                if (!isDeviceAutomation &&
+                    (!message.availability || !message.availability[0].topic.startsWith(baseTopic))) {
+                    return;
+                }
+            } catch (e) {
+                return;
+            }
+
+            const ieeeAddr = discoveryMatch[2];
+            const resolvedEntity = this.zigbee.resolveEntity(ieeeAddr);
+            let clear = !resolvedEntity || !resolvedEntity.definition;
+
+            // Only save when topic matches otherwise config is not updated when renamed by editing configuration.yaml
+            if (resolvedEntity) {
+                const key = `${discoveryMatch[3].substring(0, discoveryMatch[3].indexOf('_'))}`;
+                const triggerTopic = `${settings.get().mqtt.base_topic}/${resolvedEntity.name}/${key}`;
+                if (isDeviceAutomation && message.topic === triggerTopic) {
+                    if (!this.discoveredTriggers[ieeeAddr]) {
+                        this.discoveredTriggers[ieeeAddr] = new Set();
+                    }
+                    this.discoveredTriggers[ieeeAddr].add(discoveryMatch[3]);
+                }
+            }
+
+            if (!clear && !isDeviceAutomation) {
+                const type = discoveryMatch[1];
+                const objectID = discoveryMatch[3];
+                clear = !this.getConfigs(resolvedEntity).find((c) => c.type === type && c.object_id === objectID);
+            }
+            // Device was flagged to be excluded from homeassistant discovery
+            clear = clear || (resolvedEntity.settings.hasOwnProperty('homeassistant') &&
+                                !resolvedEntity.settings.homeassistant);
+
+            if (clear) {
+                logger.debug(`Clearing Home Assistant config '${topic}'`);
+                topic = topic.substring(this.discoveryTopic.length + 1);
+                this.mqtt.publish(topic, null, {retain: true, qos: 0}, this.discoveryTopic, false, false);
+            }
+        } else if ((topic === this.statusTopic || topic === defaultStatusTopic) && message.toLowerCase() === 'online') {
+            const timer = setTimeout(async () => {
+                // Publish all device states.
+                for (const device of this.zigbee.getClients()) {
+                    if (this.state.exists(device.ieeeAddr)) {
+                        this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
+                    }
+                }
+
+                clearTimeout(timer);
+            }, 30000);
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && type !== 'deviceLeave' && this.mqtt.isConnected()) {
+            this.discover(resolvedEntity);
+        }
+    }
+
+    getDevicePayload(resolvedEntity) {
+        return {
+            identifiers: [`zigbee2mqtt_${resolvedEntity.settings.ID}`],
+            name: resolvedEntity.settings.friendlyName,
+            sw_version: `Zigbee2MQTT ${zigbee2mqttVersion}`,
+            model: `${resolvedEntity.definition.description} (${resolvedEntity.definition.model})`,
+            manufacturer: resolvedEntity.definition.vendor,
+        };
+    }
+
+    getDiscoveryTopic(config, device) {
+        return `${config.type}/${device.ieeeAddr}/${config.object_id}/config`;
+    }
+
+    async publishDeviceTriggerDiscover(entity, key, value, force=false) {
+        if (entity.settings.hasOwnProperty('homeassistant') &&
+            (entity.settings.homeassistant == null || entity.settings.homeassistant.device_automation == null)) {
+            return;
+        }
+
+        const device = entity.device;
+        if (!this.discoveredTriggers[device.ieeeAddr]) {
+            this.discoveredTriggers[device.ieeeAddr] = new Set();
+        }
+
+        const discoveredKey = `${key}_${value}`;
+        if (this.discoveredTriggers[device.ieeeAddr].has(discoveredKey) && !force) {
+            return;
+        }
+
+        const config = {
+            type: 'device_automation',
+            object_id: `${key}_${value}`,
+            discovery_payload: {
+                automation_type: 'trigger',
+                type: key,
+            },
+        };
+
+        const topic = this.getDiscoveryTopic(config, device);
+        const payload = {
+            ...config.discovery_payload,
+            subtype: value,
+            payload: value,
+            topic: `${settings.get().mqtt.base_topic}/${entity.name}/${key}`,
+            device: this.getDevicePayload(entity),
+        };
+
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0}, this.discoveryTopic, false, false);
+        this.discoveredTriggers[device.ieeeAddr].add(discoveredKey);
+    }
+
+    // Only for homeassistant.test.js
+    _getMapping() {
+        return this.mapping;
+    }
+
+    _clearDiscoveredTrigger() {
+        this.discoveredTriggers = new Set();
+    }
+}
+
+module.exports = HomeAssistant;

lib/extension/legacy/bridgeLegacy.js

@@ -0,0 +1,473 @@
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const utils = require('../../util/utils');
+const assert = require('assert');
+const Extension = require('../extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+const configRegex =
+    new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
+const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
+
+class BridgeLegacy extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        // Bind functions
+        this.permitJoin = this.permitJoin.bind(this);
+        this.lastSeen = this.lastSeen.bind(this);
+        this.elapsed = this.elapsed.bind(this);
+        this.reset = this.reset.bind(this);
+        this.logLevel = this.logLevel.bind(this);
+        this.devices = this.devices.bind(this);
+        this.groups = this.groups.bind(this);
+        this.rename = this.rename.bind(this);
+        this.renameLast = this.renameLast.bind(this);
+        this.remove = this.remove.bind(this);
+        this.forceRemove = this.forceRemove.bind(this);
+        this.ban = this.ban.bind(this);
+        this.deviceOptions = this.deviceOptions.bind(this);
+        this.addGroup = this.addGroup.bind(this);
+        this.removeGroup = this.removeGroup.bind(this);
+        this.whitelist = this.whitelist.bind(this);
+        this.touchlinkFactoryReset = this.touchlinkFactoryReset.bind(this);
+
+        this.lastJoinedDeviceName = null;
+
+        // Set supported options
+        this.supportedOptions = {
+            'permit_join': this.permitJoin,
+            'last_seen': this.lastSeen,
+            'elapsed': this.elapsed,
+            'reset': this.reset,
+            'log_level': this.logLevel,
+            'devices': this.devices,
+            'groups': this.groups,
+            'devices/get': this.devices,
+            'rename': this.rename,
+            'rename_last': this.renameLast,
+            'remove': this.remove,
+            'force_remove': this.forceRemove,
+            'ban': this.ban,
+            'device_options': this.deviceOptions,
+            'add_group': this.addGroup,
+            'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
+            'whitelist': this.whitelist,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+        };
+    }
+
+    whitelist(topic, message) {
+        try {
+            const entity = settings.getEntity(message);
+            assert(entity, `Entity '${message}' does not exist`);
+            settings.whitelistDevice(entity.ID);
+            logger.info(`Whitelisted '${entity.friendlyName}'`);
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to whitelist '${message}' '${error}'`);
+        }
+    }
+
+    deviceOptions(topic, message) {
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error('Failed to parse message as JSON');
+            return;
+        }
+
+        if (!json.hasOwnProperty('friendly_name') || !json.hasOwnProperty('options')) {
+            logger.error('Invalid JSON message, should contain "friendly_name" and "options"');
+            return;
+        }
+
+        const entity = settings.getEntity(json.friendly_name);
+        assert(entity, `Entity '${json.friendly_name}' does not exist`);
+        settings.changeEntityOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
+    }
+
+    async permitJoin(topic, message) {
+        await this.zigbee.permitJoin(message.toLowerCase() === 'true');
+        this.publish();
+    }
+
+    async reset(topic, message) {
+        try {
+            await this.zigbee.reset('soft');
+            logger.info('Soft resetted ZNP');
+        } catch (error) {
+            logger.error('Soft reset failed');
+        }
+    }
+
+    lastSeen(topic, message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'last_seen'], message);
+        logger.info(`Set last_seen to ${message}`);
+    }
+
+    elapsed(topic, message) {
+        const allowed = ['true', 'false'];
+        if (!allowed.includes(message)) {
+            logger.error(`${message} is not an allowed value, possible: ${allowed}`);
+            return;
+        }
+
+        settings.set(['advanced', 'elapsed'], message === 'true');
+        logger.info(`Set elapsed to ${message}`);
+    }
+
+    logLevel(topic, message) {
+        const level = message.toLowerCase();
+        if (allowedLogLevels.includes(level)) {
+            logger.info(`Switching log level to '${level}'`);
+            logger.setLevel(level);
+        } else {
+            logger.error(`Could not set log level to '${level}'. Allowed level: '${allowedLogLevels.join(',')}'`);
+        }
+
+        this.publish();
+    }
+
+    async devices(topic, message) {
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const devices = this.zigbee.getDevices().map((device) => {
+            const payload = {
+                ieeeAddr: device.ieeeAddr,
+                type: device.type,
+                networkAddress: device.networkAddress,
+            };
+
+            if (device.type !== 'Coordinator') {
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const friendlyDevice = settings.getDevice(device.ieeeAddr);
+                payload.model = definition ? definition.model : device.modelID;
+                payload.vendor = definition ? definition.vendor : '-';
+                payload.description = definition ? definition.description : '-';
+                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
+                payload.manufacturerID = device.manufacturerID;
+                payload.manufacturerName = device.manufacturerName;
+                payload.powerSource = device.powerSource;
+                payload.modelID = device.modelID;
+                payload.hardwareVersion = device.hardwareVersion;
+                payload.softwareBuildID = device.softwareBuildID;
+                payload.dateCode = device.dateCode;
+                payload.lastSeen = device.lastSeen;
+            } else {
+                payload.friendly_name = 'Coordinator';
+                payload.softwareBuildID = coordinator.type;
+                payload.dateCode = coordinator.meta.revision.toString();
+                payload.lastSeen = Date.now();
+            }
+
+            return payload;
+        });
+
+        if (topic.split('/').pop() == 'get') {
+            this.mqtt.publish(
+                `bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false,
+            );
+        } else {
+            this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
+        }
+    }
+
+    groups(topic, message) {
+        const payload = settings.getGroups().map((g) => {
+            const group = {...g};
+            delete group.friendlyName;
+            return group;
+        });
+
+        this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
+    }
+
+    rename(topic, message) {
+        const invalid =
+            `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
+
+        let json = null;
+        try {
+            json = JSON.parse(message);
+        } catch (e) {
+            logger.error(invalid);
+            return;
+        }
+
+        // Validate message
+        if (!json.new || !json.old) {
+            logger.error(invalid);
+            return;
+        }
+
+        this._renameInternal(json.old, json.new);
+    }
+
+    renameLast(topic, message) {
+        if (!this.lastJoinedDeviceName) {
+            logger.error(`Cannot rename last joined device, no device has joined during this session`);
+            return;
+        }
+
+        this._renameInternal(this.lastJoinedDeviceName, message);
+    }
+
+    _renameInternal(from, to) {
+        try {
+            const isGroup = settings.getGroup(from) !== null;
+            settings.changeFriendlyName(from, to);
+            logger.info(`Successfully renamed - ${from} to ${to} `);
+            const entity = this.zigbee.resolveEntity(to);
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device, from, to};
+            eventData.homeAssisantRename = false;
+            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+            );
+        } catch (error) {
+            logger.error(`Failed to rename - ${from} to ${to}`);
+        }
+    }
+
+    addGroup(topic, message) {
+        let id = null;
+        let name = null;
+        try {
+            // json payload with id and friendly_name
+            const json = JSON.parse(message);
+            if (json.hasOwnProperty('id')) {
+                id = json.id;
+                name = `group_${id}`;
+            }
+            if (json.hasOwnProperty('friendly_name')) {
+                name = json.friendly_name;
+            }
+        } catch (e) {
+            // just friendly_name
+            name = message;
+        }
+
+        if (name == null) {
+            logger.error('Failed to add group, missing friendly_name!');
+            return;
+        }
+
+        const group = settings.addGroup(name, id);
+        this.zigbee.createGroup(group.ID);
+        this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
+        logger.info(`Added group '${name}'`);
+    }
+
+    removeGroup(topic, message) {
+        const name = message;
+        const entity = this.zigbee.resolveEntity(message);
+        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
+        settings.removeGroup(message);
+
+        this.mqtt.publish('bridge/log', stringify({type: `group_removed`, message}));
+        logger.info(`Removed group '${name}'`);
+    }
+
+    async forceRemove(topic, message) {
+        await this.removeForceRemoveOrBan('force_remove', message);
+    }
+
+    async remove(topic, message) {
+        await this.removeForceRemoveOrBan('remove', message);
+    }
+
+    async ban(topic, message) {
+        await this.removeForceRemoveOrBan('ban', message);
+    }
+
+    async removeForceRemoveOrBan(action, message) {
+        const entity = this.zigbee.resolveEntity(message.trim());
+        const lookup = {
+            ban: ['banned', 'Banning', 'ban'],
+            force_remove: ['force_removed', 'Force removing', 'force remove'],
+            remove: ['removed', 'Removing', 'remove'],
+        };
+
+        if (!entity) {
+            logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
+
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            return;
+        }
+
+        const cleanup = () => {
+            // Fire event
+            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+
+            // Remove from configuration.yaml
+            settings.removeDevice(entity.settings.ID);
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
+        };
+
+        try {
+            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            if (action === 'force_remove') {
+                await entity.device.removeFromDatabase();
+            } else {
+                await entity.device.removeFromNetwork();
+            }
+
+            cleanup();
+        } catch (error) {
+            logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+            // eslint-disable-next-line
+            logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
+
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
+        }
+
+        if (action === 'ban') {
+            settings.banDevice(entity.settings.ID);
+        }
+    }
+
+    async onMQTTConnected() {
+        await this.publish();
+    }
+
+    async onMQTTMessage(topic, message) {
+        if (!topic.match(configRegex)) {
+            return false;
+        }
+
+        const option = topic.match(configRegex)[1];
+
+        if (!this.supportedOptions.hasOwnProperty(option)) {
+            return false;
+        }
+
+        await this.supportedOptions[option](topic, message);
+
+        return true;
+    }
+
+    async publish() {
+        const info = await utils.getZigbee2mqttVersion();
+        const coordinator = await this.zigbee.getCoordinatorVersion();
+        const topic = `bridge/config`;
+        const payload = {
+            version: info.version,
+            commit: info.commitHash,
+            coordinator,
+            network: await this.zigbee.getNetworkParameters(),
+            log_level: logger.getLevel(),
+            permit_join: await this.zigbee.getPermitJoin(),
+        };
+
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceName = resolvedEntity.name;
+        }
+
+        if (type === 'deviceJoined') {
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+            );
+        } else if (type === 'deviceInterview') {
+            if (data.status === 'successful') {
+                if (resolvedEntity.definition) {
+                    const {vendor, description, model} = resolvedEntity.definition;
+                    const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                    );
+                } else {
+                    const meta = {friendly_name: resolvedEntity.name, supported: false};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_successful', meta}),
+                    );
+                }
+            } else if (data.status === 'failed') {
+                const meta = {friendly_name: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `pairing`, message: 'interview_failed', meta}),
+                );
+            } else {
+                /* istanbul ignore else */
+                if (data.status === 'started') {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `pairing`, message: 'interview_started', meta}),
+                    );
+                }
+            }
+        } else if (type === 'deviceAnnounce') {
+            const meta = {friendly_name: resolvedEntity.name};
+            this.mqtt.publish('bridge/log', stringify({type: `device_announced`, message: 'announce', meta}));
+        } else {
+            /* istanbul ignore else */
+            if (type === 'deviceLeave') {
+                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const meta = {friendly_name: name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `device_removed`, message: 'left_network', meta}),
+                );
+            }
+        }
+    }
+
+    async touchlinkFactoryReset() {
+        logger.info('Starting touchlink factory reset...');
+        this.mqtt.publish(
+            'bridge/log',
+            stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+        );
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+            );
+        } else {
+            logger.warn('Failed to factory reset device through Touchlink');
+            this.mqtt.publish(
+                'bridge/log',
+                stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+            );
+        }
+    }
+}
+
+module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -0,0 +1,50 @@
+/* istanbul ignore file */
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const Extension = require('../extension');
+
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
+
+class DeviceGroupMembership extends Extension {
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const entity = this.zigbee.resolveEntity(match[1]);
+        if (!entity || entity.type !== 'device') {
+            logger.error(`Device '${match[1]}' does not exist`);
+            return;
+        }
+
+        const response = await entity.endpoint.command(
+            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
+        );
+
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendlyName : gid;
+        });
+
+        const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
+        let msgCapacity;
+        if (capacity === 254) {
+            msgCapacity = 'it can be a part of at least 1 more group';
+        } else {
+            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
+        }
+        logger.info(`${msgGroupList} and ${msgCapacity}`);
+
+        this.publishEntityState(entity.device.ieeeAddr, {group_list: grouplist, group_capacity: capacity});
+    }
+}
+
+module.exports = DeviceGroupMembership;

lib/extension/legacy/report.js

@@ -0,0 +1,193 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
+const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
+
+const defaultConfiguration = {
+    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
+};
+
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0115'),
+];
+
+const reportKey = 1;
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const clusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
+    ],
+};
+
+class Report extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.queue = new Set();
+        this.failed = new Set();
+        this.enabled = settings.get().advanced.report;
+    }
+
+    shouldIgnoreClusterForDevice(cluster, definition) {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
+        }
+
+        return false;
+    }
+
+    async setupReporting(resolvedEntity) {
+        const {device, definition} = resolvedEntity;
+
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
+
+        const term1 = this.enabled ? 'Setup' : 'Disable';
+        const term2 = this.enabled ? 'setup' : 'disabled';
+
+        try {
+            for (const ep of device.endpoints) {
+                for (const [cluster, configuration] of Object.entries(clusters)) {
+                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
+                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                const toAdd = {...entry};
+                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
+                                items.push(toAdd);
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
+                        this.enabled ?
+                            await ep.bind(cluster, this.coordinatorEndpoint) :
+                            await ep.unbind(cluster, this.coordinatorEndpoint);
+
+                        await ep.configureReporting(cluster, items);
+                        logger.info(
+                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                        );
+                    }
+                }
+            }
+
+            if (this.enabled) {
+                device.meta.reporting = reportKey;
+            } else {
+                delete device.meta.reporting;
+                this.eventBus.emit('reportingDisabled', {device});
+            }
+
+            this.eventBus.emit(`devicesChanged`);
+        } catch (error) {
+            logger.error(
+                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
+            );
+
+            this.failed.add(device.ieeeAddr);
+        }
+
+        device.save();
+        this.queue.delete(device.ieeeAddr);
+    }
+
+    shouldSetupReporting(resolvedEntity, messageType) {
+        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
+            messageType === 'deviceLeave') return false;
+
+        const {device, definition} = resolvedEntity;
+        // Handle messages of type endDeviceAnnce and devIncoming.
+        // This message is typically send when a device comes online after being powered off
+        // Ikea TRADFRI tend to forget their reporting after powered off.
+        // Re-setup reporting.
+        // Only resetup reporting if configuredReportings was not populated yet,
+        // else reconfigure is done in zigbee-herdsman-converters ikea.js/bulbOnEvent
+        // configuredReportings are saved since Zigbee2MQTT 1.17.0
+        // https://github.com/Koenkk/zigbee2mqtt/issues/966
+        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device) &&
+            device.endpoints.filter((e) => e.configuredReportings.length === 0).length === device.endpoints.length) {
+            return true;
+        }
+
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
+
+        if (resolvedEntity.device.interviewing === true) return false;
+        if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
+        // Gledopto devices don't support reporting.
+        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+
+        if (this.enabled && device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) {
+            return false;
+        }
+
+        if (!this.enabled && !device.meta.hasOwnProperty('reporting')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    async onZigbeeStarted() {
+        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
+
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.shouldSetupReporting(resolvedEntity, null)) {
+                await this.setupReporting(resolvedEntity);
+            }
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (this.shouldSetupReporting(resolvedEntity, type)) {
+            await this.setupReporting(resolvedEntity);
+        }
+    }
+}
+
+module.exports = Report;

lib/extension/legacy/softReset.js

@@ -0,0 +1,66 @@
+/* istanbul ignore file */
+// DEPRECATED
+const settings = require('../../util/settings');
+const logger = require('../../util/logger');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
+
+/**
+ * This extensions soft resets the ZNP after a certain timeout.
+ */
+class SoftReset extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.timer = null;
+        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+    }
+
+    onZigbeeStarted() {
+        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        this.resetTimer();
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    resetTimer() {
+        if (this.timeout === 0) {
+            return;
+        }
+
+        this.clearTimer();
+        this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
+    }
+
+    async handleTimeout() {
+        logger.warn('Soft reset timeout triggered');
+
+        try {
+            await this.zigbee.reset('soft');
+            logger.warn('Soft resetted ZNP due to timeout');
+        } catch (error) {
+            logger.warn('Soft reset failed, trying stop/start');
+
+            await this.zigbee.stop();
+            logger.warn('Zigbee stopped');
+
+            try {
+                await this.zigbee.start();
+            } catch (error) {
+                logger.error('Failed to restart!');
+            }
+        }
+
+        this.resetTimer();
+    }
+
+    onZigbeeEvent() {
+        this.resetTimer();
+    }
+}
+
+module.exports = SoftReset;

lib/extension/networkMap.js

@@ -0,0 +1,302 @@
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const logger = require('../util/logger');
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension creates a network map
+ */
+class NetworkMap extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        this.legacyApi = settings.get().advanced.legacy_api;
+        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
+
+        // Bind
+        this.raw = this.raw.bind(this);
+        this.graphviz = this.graphviz.bind(this);
+        this.plantuml = this.plantuml.bind(this);
+
+        // Set supported formats
+        this.supportedFormats = {
+            'raw': this.raw,
+            'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
+        };
+    }
+
+    async onMQTTMessage(topic, message) {
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(message)) {
+                const includeRoutes = topic === this.legacyTopicRoutes;
+                const topology = await this.networkScan(includeRoutes);
+                let converted = this.supportedFormats[message](topology);
+                converted = message === 'raw' ? stringify(converted) : converted;
+                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+            }
+        }
+
+        if (topic === this.topic) {
+            try {
+                message = utils.parseJSON(message, message);
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
+        }
+    }
+
+    raw(topology) {
+        return topology;
+    }
+
+    graphviz(topology) {
+        const colors = settings.get().map_options.graphviz.colors;
+
+        let text = 'digraph G {\nnode[shape=record];\n';
+        let style = '';
+
+        topology.nodes.forEach((node) => {
+            const labels = [];
+
+            // Add friendly name
+            labels.push(`${node.friendlyName}`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            labels.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                if (node.definition) {
+                    labels.push(`${node.definition.vendor} ${node.definition.description} (${node.definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    labels.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+
+            labels.push(lastSeen);
+
+            // Shape the record according to device type
+            if (node.type == 'Coordinator') {
+                style = `style="bold, filled", fillcolor="${colors.fill.coordinator}", ` +
+                    `fontcolor="${colors.font.coordinator}"`;
+            } else if (node.type == 'Router') {
+                style = `style="rounded, filled", fillcolor="${colors.fill.router}", ` +
+                    `fontcolor="${colors.font.router}"`;
+            } else {
+                style = `style="rounded, dashed, filled", fillcolor="${colors.fill.enddevice}", ` +
+                    `fontcolor="${colors.font.enddevice}"`;
+            }
+
+            // Add the device with its labels to the graph as a node.
+            text += `  "${node.ieeeAddr}" [`+style+`, label="{${labels.join('|')}}"];\n`;
+
+            /**
+             * Add an edge between the device and its child to the graph
+             * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+             * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+             */
+            topology.links.filter((e) => (e.source.ieeeAddr === node.ieeeAddr)).forEach((e) => {
+                const lineStyle = (node.type=='EndDevice') ? 'penwidth=1, ' :
+                    (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
+                const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
+                    `weight=1, color="${colors.line.active}", `;
+                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
+                    `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
+                text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
+                text += ` [${lineStyle}${lineWeight}${lineLabels}]\n`;
+            });
+        });
+
+        text += '}';
+
+        return text.replace(/\0/g, '');
+    }
+
+    plantuml(topology) {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${node.networkAddress})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
+    async networkScan(includeRoutes) {
+        logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
+        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
+        const lqis = new Map();
+        const routingTables = new Map();
+        const failed = new Map();
+
+        for (const device of devices.filter((d) => d.type != 'EndDevice')) {
+            failed.set(device, []);
+            await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            try {
+                const result = await device.lqi();
+                lqis.set(device, result);
+                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
+            } catch (error) {
+                failed.get(device).push('lqi');
+                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
+            }
+
+            if (includeRoutes) {
+                try {
+                    const result = await device.routingTable();
+                    routingTables.set(device, result);
+                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
+                } catch (error) {
+                    failed.get(device).push('routingTable');
+                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
+                }
+            }
+        }
+
+        logger.info(`Network scan finished`);
+
+        const networkMap = {nodes: [], links: []};
+        // Add nodes
+        for (const device of devices) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            const definition = resolvedEntity.definition ? {
+                model: resolvedEntity.definition.model,
+                vendor: resolvedEntity.definition.vendor,
+                description: resolvedEntity.definition.description,
+                supports: Array.from(new Set((resolvedEntity.definition.exposes).map((e) => {
+                    return e.hasOwnProperty('name') ?
+                        e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
+            } : null;
+
+            networkMap.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
+                networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
+                modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+                definition,
+            });
+        }
+
+        // Add links
+        lqis.forEach((lqi, device) => {
+            for (const neighbor of lqi.neighbors) {
+                if (neighbor.relationship > 3) {
+                    // Relationship is not active, skip it
+                    continue;
+                }
+
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.getDeviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
+                const link = {
+                    source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
+                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
+                    linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
+                    // DEPRECATED:
+                    sourceIeeeAddr: neighbor.ieeeAddr, targetIeeeAddr: device.ieeeAddr,
+                    sourceNwkAddr: neighbor.networkAddress, lqi: neighbor.linkquality,
+                    relationship: neighbor.relationship,
+                };
+
+                const routingTable = routingTables.get(device);
+                if (routingTable) {
+                    link.routes = routingTable.table
+                        .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
+                }
+
+                networkMap.links.push(link);
+            }
+        });
+
+        return networkMap;
+    }
+}
+
+module.exports = NetworkMap;

lib/extension/onEvent.js

@@ -0,0 +1,38 @@
+const Extension = require('./extension');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+class OnEvent extends Extension {
+    async onZigbeeStarted() {
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'start', {});
+        }
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        if (resolvedEntity && resolvedEntity.type === 'device') {
+            this.callOnEvent(resolvedEntity, type, data);
+        }
+    }
+
+    async stop() {
+        super.stop();
+        for (const device of this.zigbee.getClients()) {
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            this.callOnEvent(resolvedEntity, 'stop', {});
+        }
+    }
+
+    callOnEvent(resolvedEntity, type, data) {
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
+
+        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
+        }
+    }
+}
+
+module.exports = OnEvent;

lib/extension/otaUpdate.js

@@ -0,0 +1,267 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const utils = require('../util/utils');
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const tradfriOTA = require('zigbee-herdsman-converters/lib/ota/tradfri');
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
+const Extension = require('./extension');
+
+class OTAUpdate extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.inProgress = new Set();
+        this.lastChecked = {};
+        this.legacyApi = settings.get().advanced.legacy_api;
+
+        if (settings.get().advanced.ikea_ota_use_test_url) {
+            tradfriOTA.useTestURL();
+        }
+    }
+
+    onMQTTConnected() {
+        for (const device of this.zigbee.getClients()) {
+            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+            // remove them.
+            this.removeProgressAndRemainingFromState(device);
+        }
+    }
+
+    removeProgressAndRemainingFromState(device) {
+        this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
+        this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (settings.get().ota.disable_automatic_update_check) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+
+        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        if (supportsOTA) {
+            // When a device does a next image request, it will usually do it a few times after each other
+            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
+            // each time, so this interval can be set by the user. The default is 10 minutes.
+            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
+            const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
+            if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
+
+            this.lastChecked[data.device.ieeeAddr] = Date.now();
+            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+            this.publishEntityState(data.device.ieeeAddr, payload);
+
+            if (available) {
+                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                logger.info(message);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message, meta}),
+                    );
+                }
+            }
+        }
+
+        // Respond to the OTA request:
+        // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
+        // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
+        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        if (endpoint) {
+            // Some devices send OTA requests without defining OTA cluster as input cluster.
+            await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
+        }
+    }
+
+    async readSoftwareBuildIDAndDateCode(device, update) {
+        try {
+            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
+
+            if (update) {
+                device.softwareBuildID = result.swBuildId;
+                device.dateCode = result.dateCode;
+                device.save();
+            }
+
+            return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
+        } catch (e) {
+            return null;
+        }
+    }
+
+    getEntityPublishPayload(state, progress=null, remaining=null) {
+        const payload = {update: {state}};
+        if (progress !== null) payload.update.progress = progress;
+        if (remaining !== null) payload.update.remaining = Math.round(remaining);
+
+        /* istanbul ignore else */
+        if (this.legacyApi) {
+            payload.update_available = state === 'available';
+        }
+
+        return payload;
+    }
+
+    async onMQTTMessage(topic, message) {
+        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
+            return null;
+        }
+
+        message = utils.parseJSON(message, message);
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+        const resolvedEntity = this.zigbee.resolveEntity(ID);
+        const type = topic.substring(topic.lastIndexOf('/') + 1);
+        const responseData = {id: ID};
+        let error = null;
+
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            error = `Device '${ID}' does not exist`;
+        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
+
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `ota_update`, message: error, meta}),
+                );
+            }
+        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else {
+            this.inProgress.add(resolvedEntity.device.ieeeAddr);
+
+            if (type === 'check') {
+                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
+                        resolvedEntity.device, logger,
+                    );
+                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    logger.info(msg);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: msg, meta}),
+                        );
+                    }
+
+                    const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
+                }
+            } else { // type === 'update'
+                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                logger.info(msg);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress, remaining) => {
+                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        const payload = this.getEntityPublishPayload('updating', progress, remaining);
+                        this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+                    this.eventBus.emit(`devicesChanged`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
+                    }
+                } catch (e) {
+                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('available');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
+                    }
+                }
+            }
+
+            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+        }
+
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        }
+    }
+}
+
+module.exports = OTAUpdate;

lib/extension/publish.js

@@ -0,0 +1,293 @@
+
+const settings = require('../util/settings');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const assert = require('assert');
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
+
+const groupConverters = [
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
+    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
+    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
+    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
+    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
+];
+
+class EntityPublish extends Extension {
+    parseTopic(topic) {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
+    }
+
+    async onMQTTMessage(topic, message) {
+        topic = this.parseTopic(topic);
+        if (!topic) {
+            return false;
+        }
+
+        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
+        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
+
+        if (!resolvedEntity) {
+            /* istanbul ignore else */
+            if (settings.get().advanced.legacy_api) {
+                const message = {friendly_name: entityKey};
+                this.mqtt.publish(
+                    'bridge/log',
+                    stringify({type: `entity_not_found`, message}),
+                );
+            }
+
+            logger.error(`Entity '${entityKey}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        let converters = null;
+        let target = null;
+        let options = {};
+        let device = null;
+        let definition = null;
+        let membersState = null;
+
+        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
+        if (resolvedEntity.type === 'device') {
+            if (!resolvedEntity.definition) {
+                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
+                return;
+            }
+
+            device = resolvedEntity.device;
+            definition = resolvedEntity.definition;
+            target = resolvedEntity.endpoint;
+            converters = resolvedEntity.definition.toZigbee;
+            options = resolvedEntity.settings;
+        } else {
+            target = resolvedEntity.group;
+            options = resolvedEntity.settings;
+            definition = resolvedEntity.group.members
+                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
+            converters = new Set(groupConverters);
+            for (const d of definition) {
+                d.toZigbee.forEach(converters.add, converters);
+            }
+            converters = [...converters];
+            membersState = {};
+            for (const member of resolvedEntity.group.members) {
+                const ieeeAddr = member.getDevice().ieeeAddr;
+                membersState[ieeeAddr] = this.state.get(ieeeAddr);
+            }
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        let json = {};
+        if (topic.hasOwnProperty('attribute') && topic.attribute) {
+            try {
+                json[topic.attribute] = JSON.parse(message);
+            } catch (e) {
+                json[topic.attribute] = message;
+            }
+        } else {
+            try {
+                json = JSON.parse(message);
+            } catch (e) {
+                if (stateValues.includes(message.toLowerCase())) {
+                    json = {state: message};
+                } else {
+                    logger.error(`Invalid JSON '${message}', skipping...`);
+                    return false;
+                }
+            }
+        }
+        if (!json) {
+            logger.error(`Invalid JSON '${message}', skipping...`);
+            return;
+        }
+
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
+        if (settings.get().homeassistant) {
+            const hasColorTemp = json.hasOwnProperty('color_temp');
+            const hasColor = json.hasOwnProperty('color');
+            const hasBrightness = json.hasOwnProperty('brightness');
+            const isOn = entityState && entityState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete json.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
+        const entries = Object.entries(json);
+        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters = {};
+        for (let [key, value] of entries) {
+            let endpointName = topic.endpointName;
+            let actualTarget = target;
+
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            if (resolvedEntity.type === 'device' && key.includes('_')) {
+                const endpointNameMatch = utils.getEndpointNames().find((n) => key.endsWith(`_${n}`));
+                if (endpointNameMatch) {
+                    endpointName = endpointNameMatch;
+                    key = key.replace(`_${endpointNameMatch}`, '');
+
+                    const device = target.getDevice();
+                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
+                    if (!actualTarget) {
+                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
+                        continue;
+                    }
+                }
+            }
+
+            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (topic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${json[key]})`);
+                continue;
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {
+                endpoint_name: endpointName,
+                options,
+                message: {...json},
+                logger,
+                device,
+                state: entityState,
+                membersState,
+                mapped: definition,
+            };
+
+            // Strip endpoint name from meta.message properties.
+            if (meta.endpoint_name) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(meta.endpoint_name)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - meta.endpoint_name.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
+            try {
+                if (topic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
+                    const result = await converter.convertSet(actualTarget, key, value, meta);
+                    const isGroup = actualTarget.constructor.name == 'Group';
+                    // Optimistic behaviour of groups and devices is different.
+                    const publishOptimistic = isGroup || (!options.hasOwnProperty('optimistic') || options.optimistic);
+                    if (result && result.state && publishOptimistic) {
+                        const msg = result.state;
+
+                        if (endpointName) {
+                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
+                                if (msg.hasOwnProperty(key)) {
+                                    msg[`${key}_${endpointName}`] = msg[key];
+                                    delete msg[key];
+                                }
+                            }
+                        }
+
+                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                    }
+
+                    if (result && result.membersState && publishOptimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            this.publishEntityState(ieeeAddr, state);
+                        }
+                    }
+
+                    // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+                    // So here we re-read the value after a specified timeout, this timeout could for example be the
+                    // transition time of a color change or for forcing a state read for devices that don't
+                    // automatically report a new state when set.
+                    // When reporting is requested for a device (report: true in device-specific settings) we won't
+                    // ever issue a read here, as we assume the device will properly report changes.
+                    // Only do this when the retrieve_state option is enabled for this device.
+                    // retrieve_state == decprecated
+                    if (
+                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
+                        resolvedEntity.settings.retrieve_state
+                    ) {
+                        setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
+                    }
+                } else if (topic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
+                    await converter.convertGet(actualTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const meta = {friendly_name: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `zigbee_publish_error`, message, meta}),
+                    );
+                }
+            }
+
+            usedConverters[endpointOrGroupID].push(converter);
+        }
+
+        return true;
+    }
+}
+
+module.exports = EntityPublish;

lib/extension/receive.js

@@ -0,0 +1,173 @@
+const settings = require('../util/settings');
+const logger = require('../util/logger');
+const utils = require('../util/utils');
+const debounce = require('debounce');
+const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+class Receive extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.elapsed = {};
+        this.debouncers = {};
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+    }
+
+    async onZigbeeStarted() {
+        this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+    }
+
+    async onPublishEntityState(data) {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce') {
+            for (const key of Object.keys(data.payload)) {
+                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
+            }
+        }
+    }
+
+    publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
+        if (!this.debouncers[ieeeAddr]) {
+            this.debouncers[ieeeAddr] = {
+                payload: {},
+                publish: debounce(() => {
+                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
+                    this.debouncers[ieeeAddr].payload = {};
+                }, time * 1000),
+            };
+        }
+
+        if (this.isPayloadConflicted(payload, this.debouncers[ieeeAddr].payload, debounceIgnore)) {
+            // publish previous payload immediately
+            this.debouncers[ieeeAddr].publish.flush();
+        }
+
+        // extend debounced payload with current
+        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
+        this.debouncers[ieeeAddr].publish();
+    }
+
+    // if debounce_ignore are specified (Array of strings)
+    // then all newPayload values with key present in debounce_ignore
+    // should equal or be undefined in oldPayload
+    // otherwise payload is conflicted
+    isPayloadConflicted(newPayload, oldPayload, debounceIgnore) {
+        let result = false;
+        Object.keys(oldPayload)
+            .filter((key) => (debounceIgnore || []).includes(key))
+            .forEach((key) => {
+                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
+                    result = true;
+                }
+            });
+
+        return result;
+    }
+
+    shouldProcess(type, data, resolvedEntity) {
+        if (type !== 'message' || !resolvedEntity) {
+            return false;
+        }
+
+        if (!resolvedEntity.definition) {
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
+            return false;
+        }
+
+        return true;
+    }
+
+    onZigbeeEvent(type, data, resolvedEntity) {
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
+            resolvedEntity = this.zigbee.resolveEntity(data.device);
+        }
+
+        if (!this.shouldProcess(type, data, resolvedEntity)) {
+            return;
+        }
+
+        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
+            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
+            return c.cluster === data.cluster && type;
+        });
+
+        // Check if there is an available converter, genOta messages are not interesting.
+        if (!converters.length && !['genOta', 'genTime', 'genBasic'].includes(data.cluster)) {
+            logger.debug(
+                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
+                `and type '${data.type}' and data '${stringify(data.data)}'`,
+            );
+            return;
+        }
+
+        // Convert this Zigbee message to a MQTT message.
+        // Get payload for the message.
+        // - If a payload is returned publish it to the MQTT broker
+        // - If NO payload is returned do nothing. This is for non-standard behaviour
+        //   for e.g. click switches where we need to count number of clicks and detect long presses.
+        const publish = (payload) => {
+            if (settings.get().advanced.elapsed) {
+                const now = Date.now();
+                if (this.elapsed[data.device.ieeeAddr]) {
+                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
+                }
+
+                this.elapsed[data.device.ieeeAddr] = now;
+            }
+
+            // Check if we have to debounce
+            if (resolvedEntity.settings.debounce) {
+                this.publishDebounce(
+                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
+                    resolvedEntity.settings.debounce_ignore,
+                );
+            } else {
+                this.publishEntityState(data.device.ieeeAddr, payload);
+            }
+        };
+
+        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
+        let payload = {};
+        converters.forEach((converter) => {
+            const converted = converter.convert(
+                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
+            );
+            if (converted) {
+                payload = {...payload, ...converted};
+            }
+        });
+
+        if (Object.keys(payload).length) {
+            publish(payload);
+        }
+    }
+}
+
+module.exports = Receive;

lib/homeassistant.js

@@ -1,355 +0,0 @@
-const settings = require('./util/settings');
-
-const configurations = {
-    // Binary sensor
-    'binary_sensor_occupancy': {
-        type: 'binary_sensor',
-        object_id: 'occupancy',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.occupancy }}',
-            device_class: 'motion',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'binary_sensor_contact': {
-        type: 'binary_sensor',
-        object_id: 'contact',
-        discovery_payload: {
-            payload_on: false,
-            payload_off: true,
-            value_template: '{{ value_json.contact }}',
-            device_class: 'door',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'binary_sensor_water_leak': {
-        type: 'binary_sensor',
-        object_id: 'water_leak',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.water_leak }}',
-            device_class: 'moisture',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'binary_sensor_smoke': {
-        type: 'binary_sensor',
-        object_id: 'smoke',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.smoke }}',
-            device_class: 'smoke',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'binary_sensor_router': {
-        type: 'binary_sensor',
-        object_id: 'router',
-        discovery_payload: {
-            payload_on: true,
-            payload_off: false,
-            value_template: '{{ value_json.state }}',
-            device_class: 'connectivity',
-            json_attributes: ['description', 'type', 'rssi'],
-        },
-    },
-
-    // Sensor
-    'sensor_illuminance': {
-        type: 'sensor',
-        object_id: 'illuminance',
-        discovery_payload: {
-            unit_of_measurement: 'lx',
-            icon: 'mdi:theme-light-dark',
-            value_template: '{{ value_json.illuminance }}',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'sensor_humidity': {
-        type: 'sensor',
-        object_id: 'humidity',
-        discovery_payload: {
-            unit_of_measurement: '%',
-            icon: 'mdi:water-percent',
-            value_template: '{{ value_json.humidity }}',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'sensor_temperature': {
-        type: 'sensor',
-        object_id: 'temperature',
-        discovery_payload: {
-            unit_of_measurement: '°C',
-            icon: 'mdi:temperature-celsius',
-            value_template: '{{ value_json.temperature }}',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'sensor_pressure': {
-        type: 'sensor',
-        object_id: 'pressure',
-        discovery_payload: {
-            unit_of_measurement: 'Pa',
-            icon: 'mdi:speedometer',
-            value_template: '{{ value_json.pressure }}',
-            json_attributes: ['battery', 'voltage'],
-        },
-    },
-    'sensor_click': {
-        type: 'sensor',
-        object_id: 'click',
-        discovery_payload: {
-            icon: 'mdi:toggle-switch',
-            value_template: '{{ value_json.click }}',
-            json_attributes: ['battery', 'voltage', 'action', 'duration'],
-        },
-    },
-    'sensor_power': {
-        type: 'sensor',
-        object_id: 'power',
-        discovery_payload: {
-            unit_of_measurement: 'Watt',
-            icon: 'mdi:flash',
-            value_template: '{{ value_json.power }}',
-            json_attributes: ['voltage', 'temperature', 'consumption', 'current', 'power_factor'],
-        },
-    },
-    'sensor_action': {
-        type: 'sensor',
-        object_id: 'action',
-        discovery_payload: {
-            icon: 'mdi:cube',
-            value_template: '{{ value_json.action }}',
-            json_attributes: ['battery', 'voltage', 'angle', 'side', 'from_side', 'to_side'],
-        },
-    },
-    'sensor_brightness': {
-        type: 'sensor',
-        object_id: 'brightness',
-        discovery_payload: {
-            unit_of_measurement: 'brightness',
-            icon: 'mdi:brightness-5',
-            value_template: '{{ value_json.brightness }}',
-            json_attributes: [],
-        },
-    },
-
-    // Light
-    'light_brightness_colortemp_xy': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            color_temp: true,
-            xy: true,
-            platform: 'mqtt_json',
-            command_topic: true,
-        },
-    },
-    'light_brightness_xy': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            xy: true,
-            platform: 'mqtt_json',
-            command_topic: true,
-        },
-    },
-    'light_brightness_colortemp': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            color_temp: true,
-            platform: 'mqtt_json',
-            command_topic: true,
-        },
-    },
-    'light_brightness': {
-        type: 'light',
-        object_id: 'light',
-        discovery_payload: {
-            brightness: true,
-            platform: 'mqtt_json',
-            command_topic: true,
-        },
-    },
-
-    // Switch
-    'switch': {
-        type: 'switch',
-        object_id: 'switch',
-        discovery_payload: {
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            value_template: '{{ value_json.state }}',
-            command_topic: true,
-        },
-    },
-    'switch_left': {
-        type: 'switch',
-        object_id: 'switch_left',
-        discovery_payload: {
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            value_template: '{{ value_json.state_left }}',
-            command_topic: true,
-            command_topic_prefix: 'left',
-        },
-    },
-    'switch_right': {
-        type: 'switch',
-        object_id: 'switch_right',
-        discovery_payload: {
-            payload_off: 'OFF',
-            payload_on: 'ON',
-            value_template: '{{ value_json.state_right }}',
-            command_topic: true,
-            command_topic_prefix: 'right',
-        },
-    },
-};
-
-// Map homeassitant configurations to devices.
-const mapping = {
-    'WXKG01LM': [configurations.sensor_click],
-    'WXKG11LM': [configurations.sensor_click],
-    'WXKG12LM': [configurations.sensor_click],
-    'WXKG03LM': [configurations.sensor_click],
-    'WXKG02LM': [configurations.sensor_click],
-    'QBKG04LM': [configurations.switch],
-    'QBKG03LM': [configurations.switch_left, configurations.switch_right],
-    'WSDCGQ01LM': [configurations.sensor_temperature, configurations.sensor_humidity],
-    'WSDCGQ11LM': [configurations.sensor_temperature, configurations.sensor_humidity, configurations.sensor_pressure],
-    'RTCGQ01LM': [configurations.binary_sensor_occupancy],
-    'RTCGQ11LM': [configurations.binary_sensor_occupancy, configurations.sensor_illuminance],
-    'MCCGQ01LM': [configurations.binary_sensor_contact],
-    'MCCGQ11LM': [configurations.binary_sensor_contact],
-    'SJCGQ11LM': [configurations.binary_sensor_water_leak],
-    'MFKZQ01LM': [configurations.sensor_action],
-    'ZNCZ02LM': [configurations.switch, configurations.sensor_power],
-    'QBCZ11LM': [configurations.switch, configurations.sensor_power],
-    'LED1545G12': [configurations.light_brightness_colortemp],
-    'LED1623G12': [configurations.light_brightness],
-    'LED1622G12': [configurations.light_brightness],
-    'LED1537R6': [configurations.light_brightness_colortemp],
-    'LED1650R5': [configurations.light_brightness],
-    'LED1536G5': [configurations.light_brightness_colortemp],
-    '7146060PH': [configurations.light_brightness_colortemp_xy],
-    'F7C033': [configurations.light_brightness],
-    'JTYJ-GD-01LM/BW': [configurations.binary_sensor_smoke],
-    'PLUG EDP RE:DY': [configurations.switch, configurations.sensor_power],
-    'CC2530.ROUTER': [configurations.binary_sensor_router],
-    'AA70155': [configurations.light_brightness_colortemp],
-    'AA69697': [configurations.light_brightness_colortemp_xy],
-    'HALIGHTDIMWWE27': [configurations.light_brightness],
-    'AB3257001NJ': [configurations.switch],
-    '8718696449691': [configurations.light_brightness],
-    'RB 185 C': [configurations.light_brightness_colortemp_xy],
-    '9290012573A': [configurations.light_brightness_colortemp_xy],
-    'LED1624G9': [configurations.light_brightness_xy],
-    '73742': [configurations.light_brightness_colortemp],
-    '73740': [configurations.light_brightness_colortemp],
-    '22670': [configurations.light_brightness],
-    'ICTC-G-1': [configurations.sensor_brightness],
-    'ICPSHC24-30EU-IL-1': [configurations.light_brightness],
-    '45852GE': [configurations.light_brightness],
-    'E11-G13': [configurations.light_brightness],
-    'LED1649C5': [configurations.light_brightness],
-    'ICPSHC24-10EU-IL-1': [configurations.light_brightness],
-    'LED1546G12': [configurations.light_brightness_colortemp],
-    'L1527': [configurations.light_brightness_colortemp],
-    'L1529': [configurations.light_brightness_colortemp],
-    'L1528': [configurations.light_brightness_colortemp],
-    'RB 165': [configurations.light_brightness],
-    'RB 175 W': [configurations.light_brightness],
-    'RS 125': [configurations.light_brightness],
-    'RB 145': [configurations.light_brightness],
-    'PL 110': [configurations.light_brightness],
-    'ST 110': [configurations.light_brightness],
-    'UC 110': [configurations.light_brightness],
-    'DL 110 N': [configurations.light_brightness],
-    'DL 110 W': [configurations.light_brightness],
-    'SL 110 N': [configurations.light_brightness],
-    'SL 110 M': [configurations.light_brightness],
-    'SL 110 W': [configurations.light_brightness],
-    'AA68199': [configurations.light_brightness_colortemp],
-    'QBKG11LM': [configurations.switch, configurations.sensor_power],
-    'QBKG12LM': [configurations.switch_left, configurations.switch_right, configurations.sensor_power],
-    'K2RGBW01': [configurations.light_brightness_colortemp_xy],
-    '9290011370': [configurations.light_brightness],
-    'DNCKATSW001': [configurations.switch],
-    'Z809A': [configurations.switch, configurations.sensor_power],
-    'NL08-0800': [configurations.light_brightness],
-    '915005106701': [configurations.light_brightness_colortemp_xy],
-    'AB32840': [configurations.light_brightness_colortemp],
-};
-
-// A map of all discoverd devices
-const discovered = {};
-
-function discover(deviceID, model, mqtt, force) {
-    // Check if already discoverd and check if there are configs.
-    const discover = force || !discovered[deviceID];
-    if (!discover || !mapping[model] || !settings.getDevice(deviceID)) {
-        return;
-    }
-
-    const friendlyName = settings.getDevice(deviceID).friendly_name;
-
-    mapping[model].forEach((config) => {
-        const topic = `${config.type}/${deviceID}/${config.object_id}/config`;
-        const payload = config.discovery_payload;
-        payload.state_topic = `${settings.get().mqtt.base_topic}/${friendlyName}`;
-        payload.availability_topic = `${settings.get().mqtt.base_topic}/bridge/state`;
-
-        // Set unique names in cases this device produces multiple entities in homeassistant.
-        payload.name = mapping[model].length > 1 ? `${friendlyName}_${config.object_id}` : friendlyName;
-
-        // Only set unique_id when user did not set a friendly_name yet,
-        // see https://github.com/Koenkk/zigbee2mqtt/issues/138
-        if (deviceID === friendlyName) {
-            payload.unique_id = `${deviceID}_${config.object_id}_${settings.get().mqtt.base_topic}`;
-        }
-
-        if (payload.command_topic) {
-            payload.command_topic = `${settings.get().mqtt.base_topic}/${friendlyName}/`;
-
-            if (payload.command_topic_prefix) {
-                payload.command_topic += `${payload.command_topic_prefix}/`;
-            }
-
-            payload.command_topic += 'set';
-        }
-
-        mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0}, null, 'homeassistant');
-    });
-
-    discovered[deviceID] = true;
-}
-
-function clear(deviceID, model, mqtt) {
-    // Check if there are configs.
-    if (!mapping[model]) {
-        return;
-    }
-
-    mapping[model].forEach((config) => {
-        const topic = `${config.type}/${deviceID}/${config.object_id}/config`;
-        const payload = '';
-        mqtt.publish(topic, payload, {retain: true, qos: 0}, null, 'homeassistant');
-    });
-
-    discovered[deviceID] = false;
-}
-
-module.exports = {
-    mapping: mapping,
-    discover: (deviceID, model, mqtt, force) => discover(deviceID, model, mqtt, force),
-    clear: (deviceID, model, mqtt) => clear(deviceID, model, mqtt),
-};

lib/mqtt.js

@@ -1,32 +1,66 @@
 const mqtt = require('mqtt');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
+const fs = require('fs');
+const events = require('events');
 
-class MQTT {
+class MQTT extends events.EventEmitter {
     constructor() {
-        this.handleConnect = this.handleConnect.bind(this);
-        this.handleMessage = this.handleMessage.bind(this);
+        super();
+        this.onMessage = this.onMessage.bind(this);
+        this.publishedTopics = new Set();
     }
 
-    connect(onMessage, subscriptions, callback) {
+    async connect() {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
-        const options = {};
-        if (mqttSettings.user && mqttSettings.password) {
-            options.username = mqttSettings.user;
-            options.password = mqttSettings.password;
+        const options = {
+            will: {
+                topic: `${settings.get().mqtt.base_topic}/bridge/state`,
+                payload: 'offline',
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
+            },
+        };
+
+        if (mqttSettings.version) {
+            options.protocolVersion = mqttSettings.version;
         }
 
-        this.client = mqtt.connect(mqttSettings.server, options);
+        if (mqttSettings.keepalive) {
+            logger.debug(`Using MQTT keepalive: ${mqttSettings.keepalive}`);
+            options.keepalive = mqttSettings.keepalive;
+        }
 
-        // Register callbacks.
-        this.client.on('connect', () => {
-            this.handleConnect();
-            callback();
-        });
+        if (mqttSettings.ca) {
+            logger.debug(`MQTT SSL/TLS: Path to CA certificate = ${mqttSettings.ca}`);
+            options.ca = fs.readFileSync(mqttSettings.ca);
+        }
 
-        this.client.on('message', this.handleMessage);
+        if (mqttSettings.key && mqttSettings.cert) {
+            logger.debug(`MQTT SSL/TLS: Path to client key = ${mqttSettings.key}`);
+            logger.debug(`MQTT SSL/TLS: Path to client certificate = ${mqttSettings.cert}`);
+            options.key = fs.readFileSync(mqttSettings.key);
+            options.cert = fs.readFileSync(mqttSettings.cert);
+        }
+
+        if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
+            options.username = mqttSettings.user;
+            options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
+        }
+
+        if (mqttSettings.client_id) {
+            logger.debug(`Using MQTT client ID: '${mqttSettings.client_id}'`);
+            options.clientId = mqttSettings.client_id;
+        }
+
+        if (mqttSettings.hasOwnProperty('reject_unauthorized') && !mqttSettings.reject_unauthorized) {
+            logger.debug(`MQTT reject_unauthorized set false, ignoring certificate warnings.`);
+            options.rejectUnauthorized = false;
+        }
 
         // Set timer at interval to check if connected to MQTT server.
         const interval = 10 * 1000; // seconds * 1000.
@@ -36,49 +70,75 @@ class MQTT {
             }
         }, interval);
 
-        this.onMessage = onMessage;
-        this.subscriptions = subscriptions;
-    }
+        return new Promise((resolve) => {
+            this.client = mqtt.connect(mqttSettings.server, options);
 
-    disconnect() {
-        clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
+            const self = this;
+            this.client.on('connect', () => {
+                logger.info('Connected to MQTT server');
+                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
+                self.publish('bridge/state', 'online', {retain: true, qos: 0});
+                resolve();
+            });
 
-        this.publish('bridge/state', 'offline', {retain: true, qos: 0}, () => {
-            logger.info('Disconnecting from MQTT server');
-            this.client.end();
+            this.client.on('message', this.onMessage);
         });
     }
 
-    handleConnect() {
-        logger.info('Connected to MQTT server');
-        this.publish('bridge/state', 'online', {retain: true, qos: 0});
-        this.subscriptions.forEach((topic) => this.client.subscribe(topic));
+    async disconnect() {
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = null;
+        await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
+        logger.info('Disconnecting from MQTT server');
+        this.client.end();
     }
 
-    handleMessage(topic, message) {
-        if (this.onMessage) {
-            this.onMessage(topic, message);
+    subscribe(topic) {
+        this.client.subscribe(topic);
+    }
+
+    onMessage(topic, message) {
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.emit('message', {topic, message: message + ''});
         }
     }
 
-    publish(topic, payload, options, callback, baseTopic) {
-        baseTopic = baseTopic ? baseTopic : settings.get().mqtt.base_topic;
-        topic = `${baseTopic}/${topic}`;
+    isConnected() {
+        return this.client && !this.client.reconnecting;
+    }
 
-        if (!this.client || this.client.reconnecting) {
-            logger.error(`Not connected to MQTT server!`);
-            logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
+        topic = `${base}/${topic}`;
+        options = {qos: 0, retain: false, ...options};
+
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
+        if (!this.isConnected()) {
+            if (!skipLog) {
+                logger.error(`Not connected to MQTT server!`);
+                logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            }
             return;
         }
 
-        logger.info(`MQTT publish, topic: '${topic}', payload: '${payload}'`);
-        this.client.publish(topic, payload, options, callback);
-    }
+        if (!skipLog) {
+            logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        }
 
-    log(type, message) {
-        const payload = {type: type, message: message};
-        this.publish('bridge/log', JSON.stringify(payload), {retain: false});
+        const actualOptions = {...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
+        return new Promise((resolve) => {
+            this.client.publish(topic, payload, actualOptions, () => {
+                this.emit('publishedMessage', {topic, payload, options});
+                resolve();
+            });
+        });
     }
 }
 

lib/state.js

@@ -0,0 +1,119 @@
+const logger = require('./util/logger');
+const data = require('./util/data');
+const settings = require('./util/settings');
+const fs = require('fs');
+const objectAssignDeep = require('object-assign-deep');
+
+const saveInterval = 1000 * 60 * 5; // 5 minutes
+
+const dontCacheProperties = [
+    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
+    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
+    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
+];
+
+class State {
+    constructor(eventBus) {
+        this.state = {};
+        this.file = data.joinPath('state.json');
+        this.timer = null;
+        this.eventBus = eventBus;
+    }
+
+    start() {
+        this._load();
+
+        // Save the state on every interval
+        this.clearTimer();
+        this.timer = setInterval(() => this.save(), saveInterval);
+    }
+
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+
+    stop() {
+        this.clearTimer();
+        this.save();
+    }
+
+    _load() {
+        if (fs.existsSync(this.file)) {
+            try {
+                this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
+                logger.debug(`Loaded state from file ${this.file}`);
+            } catch (e) {
+                logger.debug(`Failed to load state from file ${this.file} (corrupt file?)`);
+            }
+        } else {
+            logger.debug(`Can't load state from file ${this.file} (doesn't exist)`);
+        }
+    }
+
+    save() {
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
+        } else {
+            logger.debug(`Not saving state`);
+        }
+    }
+
+    exists(ID) {
+        return this.state.hasOwnProperty(ID);
+    }
+
+    get(ID) {
+        return this.state[ID];
+    }
+
+    set(ID, state, reason=null) {
+        const toState = objectAssignDeep.noMutate(state);
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
+                delete toState[property];
+            }
+        }
+
+        const fromState = this.state[ID];
+
+        this.state[ID] = toState;
+
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: state, reason});
+    }
+
+    removeKey(ID, path) {
+        if (this.exists(ID)) {
+            let state = this.state[ID];
+            for (let i = 0; i < path.length; i++) {
+                const key = path[i];
+                if (i === path.length - 1) {
+                    delete state[key];
+                } else {
+                    if (state[key]) {
+                        state = state[key];
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    remove(ID) {
+        if (this.exists(ID)) {
+            delete this.state[ID];
+        }
+    }
+}
+
+module.exports = State;

lib/util/data.js

@@ -2,13 +2,21 @@ const path = require('path');
 
 let dataPath = null;
 
-if (process.env.ZIGBEE2MQTT_DATA) {
-    dataPath = process.env.ZIGBEE2MQTT_DATA;
-} else {
-    dataPath = path.join(__dirname, '..', '..', 'data');
-    dataPath = path.normalize(dataPath);
+function load() {
+    if (process.env.ZIGBEE2MQTT_DATA) {
+        dataPath = process.env.ZIGBEE2MQTT_DATA;
+    } else {
+        dataPath = path.join(__dirname, '..', '..', 'data');
+        dataPath = path.normalize(dataPath);
+    }
 }
 
+load();
+
 module.exports = {
     joinPath: (file) => path.join(dataPath, file),
+    getPath: () => dataPath,
+
+    // For test only.
+    _reload: () => load(),
 };

lib/util/logger.js

@@ -1,37 +1,152 @@
 const winston = require('winston');
-const data = require('./data');
+const moment = require('moment');
 const settings = require('./settings');
+const path = require('path');
+const fs = require('fs');
+const fx = require('mkdir-recursive');
+const rimraf = require('rimraf');
+const colorizer = winston.format.colorize();
+const assert = require('assert');
 
-let logLevel = '';
-if (process.env.DEBUG) {
-    logLevel = 'debug';
-} else if (settings.get().advanced && settings.get().advanced.log_level) {
-    logLevel = settings.get().advanced.log_level;
-} else {
-    logLevel = winston.level.info;
+// What transports to enable
+const output = settings.get().advanced.log_output;
+
+// Directory to log to
+const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+// Make sure that log directoy exsists when not logging to stdout only
+if (output.includes('file')) {
+    fx.mkdirSync(directory);
 }
 
-const logger = new (winston.Logger)({
-    transports: [
-        new (winston.transports.File)({
-            filename: data.joinPath('log.txt'),
-            json: false,
-            level: logLevel,
-            maxFiles: 3,
-            maxsize: 10000000, // 10MB
-        }),
-        new (winston.transports.Console)({
-            timestamp: () => new Date().toLocaleString(),
-            formatter: function(options) {
-                return options.timestamp() + ' ' +
-                        winston.config.colorize(options.level, options.level.toUpperCase()) + ' ' +
-                        (options.message ? options.message : '') +
-                        (options.meta && Object.keys(options.meta).length ? '\n\t'+ JSON.stringify(options.meta) : '' );
-            },
-        }),
-    ],
-});
+const z2mToWinstonLevel = (level) => level === 'warn' ? 'warning' : level;
+const winstonToZ2mLevel = (level) => level === 'warning' ? 'warn' : level;
 
-logger.transports.console.level = logLevel;
+// Determine the log level.
+let level = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
+level = z2mToWinstonLevel(level);
+
+const levelWithCompensatedLength = {
+    'info': 'info ',
+    'error': 'error',
+    'warn': 'warn ',
+    'debug': 'debug',
+};
+
+/* istanbul ignore next */
+const timestampFormat = () => moment().format(settings.get().advanced.timestamp_format);
+
+// Setup default console logger
+const transportsToUse = [
+    new winston.transports.Console({
+        level,
+        silent: !output.includes('console'),
+        format: winston.format.combine(
+            winston.format.timestamp({format: timestampFormat}),
+            winston.format.printf(/* istanbul ignore next */(info) => {
+                let {timestamp, level, message} = info;
+                level = winstonToZ2mLevel(level);
+                const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
+                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+            }),
+        ),
+    }),
+];
+
+// Add file logger when enabled
+// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            let {timestamp, level, message} = info;
+            level = winstonToZ2mLevel(level);
+            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
+if (output.includes('file')) {
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    require('winston-syslog').Syslog;
+    const options = {
+        app_name: 'Zigbee2MQTT',
+        format: winston.format.printf(/* istanbul ignore next */(info) => {
+            return `${info.message}`;
+        }),
+        ...settings.get().advanced.log_syslog,
+    };
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
+// Create logger
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+
+// Cleanup any old log directory.
+function cleanup() {
+    if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
+        const rootDirectory = path.join(directory, '..');
+
+        let directories = fs.readdirSync(rootDirectory).map((d) => {
+            d = path.join(rootDirectory, d);
+            return {path: d, birth: fs.statSync(d).mtime};
+        });
+
+        directories.sort((a, b) => b.birth - a.birth);
+        directories = directories.slice(10, directories.length);
+        directories.forEach((dir) => {
+            logger.debug(`Removing old log directory '${dir.path}'`);
+            rimraf.sync(dir.path);
+        });
+    }
+}
+
+// Print to user what logging is enabled
+function logOutput() {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        logger.cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
+    }
+}
+
+logger.addTransport = (transport) => {
+    transport.level = transportsToUse[0].level;
+    logger.add(transport);
+};
+logger.cleanup = cleanup;
+logger.logOutput = logOutput;
+logger.getLevel = () => winstonToZ2mLevel(transportsToUse[0].level);
+logger.setLevel = (level) => {
+    level = z2mToWinstonLevel(level);
+    logger.transports.forEach((transport) => transport.level = level);
+};
+
+// winston.config.syslog.levels doesnt have warn, but is required for syslog.
+/* istanbul ignore next */
+logger.warn = (message) => logger.warning(message);
 
 module.exports = logger;

lib/util/settings.js

@@ -1,54 +1,700 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
 const data = require('./data');
-const file = data.joinPath('configuration.yaml');
+const utils = require('./utils');
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
+const objectAssignDeep = require(`object-assign-deep`);
+const path = require('path');
+const yaml = require('./yaml');
+const Ajv = require('ajv');
+const schema = require('./settings.schema.json');
+const ajvSetting = new Ajv({allErrors: true}).compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword('requiresRestart', {validate: (v) => !v}).compile(schema);
 
-let settings = read();
+const defaults = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
+    whitelist: [],
+    ban: [],
+    permit_join: false,
+    mqtt: {
+        include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
+    },
+    serial: {
+        disable_led: false,
+    },
+    device_options: {},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: '#fff8ce',
+                    coordinator: '#e04e5d',
+                    router: '#4ea3e0',
+                },
+                font: {
+                    coordinator: '#ffffff',
+                    router: '#ffffff',
+                    enddevice: '#000000',
+                },
+                line: {
+                    active: '#009900',
+                    inactive: '#994444',
+                },
+            },
+        },
+    },
+    experimental: {
+        // json or attribute or attribute_and_json
+        output: 'json',
+    },
+    advanced: {
+        legacy_api: true,
+        log_rotation: true,
+        log_output: ['console', 'file'],
+        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
+        log_file: 'log.txt',
+        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
+        soft_reset_timeout: 0,
+        pan_id: 0x1a62,
+        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
+        channel: 11,
+        adapter_concurrent: null,
+        adapter_delay: null,
 
-function writeRead() {
-    write();
-    settings = read();
-}
+        // Availability timeout in seconds, disabled by default.
+        availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
+        availability_blacklist: [],
+        availability_whitelist: [],
+
+        /**
+         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
+         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
+         *
+         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
+         * advanced.cache_state in configuration.yaml allows to configure this.
+         * https://www.zigbee2mqtt.io/configuration/configuration.html
+         */
+        cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
+
+        /**
+         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
+         * "ISO_8601": ISO 8601 format
+         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
+         * "epoch": milliseconds elapsed since the UNIX epoch
+         * "disable": no last_seen attribute (default)
+         */
+        last_seen: 'disable',
+
+        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
+        elapsed: false,
+
+        /**
+         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
+         *
+         * Network key will serve as the encryption key of your network.
+         * Changing this will require you to repair your devices.
+         */
+        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
+
+        /**
+         * Enables reporting feature
+         */
+        report: false,
+
+        /**
+         * Home Assistant discovery topic
+         */
+        homeassistant_discovery_topic: 'homeassistant',
+
+        /**
+         * Home Assistant status topic
+         */
+        homeassistant_status_topic: 'hass/status',
+
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
+        /**
+         * Configurable timestampFormat
+         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
+         */
+        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
+    },
+    ota: {
+        /**
+         * Minimal time delta in minutes between polling third party server for potential firmware updates
+         */
+        update_check_interval: 10,
+        /**
+         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
+         * If set to true, only a user-initiated update search will be possible.
+         */
+        disable_automatic_update_check: false,
+    },
+    external_converters: [],
+};
+
+let _settings;
+let _settingsWithDefaults;
 
 function write() {
-    fs.writeFileSync(file, yaml.safeDump(settings));
+    const settings = get();
+    const toWrite = objectAssignDeep.noMutate(settings);
+
+    // Read settings to check if we have to split devices/groups into separate file.
+    const actual = yaml.read(file);
+
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [
+        ['mqtt', 'user'],
+        ['mqtt', 'password'],
+        ['advanced', 'network_key'],
+        ['frontend', 'auth_token'],
+    ]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
+    }
+
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type) => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            const content = objectAssignDeep.noMutate(settings[type]);
+
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f, i) => i !== 0)
+                    .map((f) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c) => Object.keys(c))
+                    .forEach((k) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
+
+    yaml.writeIfChanged(file, toWrite);
+
+    _settings = read();
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+}
+
+function validate() {
+    try {
+        get();
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            return [
+                `Your YAML file: '${error.file}' is invalid ` +
+                `(use https://jsonformatter.org/yaml-validator to find and fix the issue)`,
+            ];
+        }
+
+        return [error.message];
+    }
+
+    if (!ajvSetting(_settings)) {
+        return ajvSetting.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+    }
+
+    const errors = [];
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    if (_settings.advanced && _settings.advanced.pan_id && typeof _settings.advanced.pan_id === 'string' &&
+        _settings.advanced.pan_id !== 'GENERATE') {
+        errors.push(`advanced.pan_id: should be number or 'GENERATE' (is '${_settings.advanced.pan_id}')`);
+    }
+
+    // Verify that all friendly names are unique
+    const names = [];
+    const check = (name) => {
+        if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
+        errors.push(...utils.validateFriendlyName(name));
+        names.push(name);
+    };
+
+    const settingsWithDefaults = getWithDefaults();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                errors.push('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list, type) => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                errors.push(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
+
+    return errors;
 }
 
 function read() {
-    return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const s = yaml.read(file);
+
+    // Read !secret MQTT username and password if set
+    const interpetValue = (value) => {
+        const re = /!(.*) (.*)/g;
+        const match = re.exec(value);
+        if (match) {
+            const file = data.joinPath(`${match[1]}.yaml`);
+            const key = match[2];
+            return yaml.read(file)[key];
+        } else {
+            return value;
+        }
+    };
+
+    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+        s.mqtt.user = interpetValue(s.mqtt.user);
+        s.mqtt.password = interpetValue(s.mqtt.password);
+    }
+
+    if (s.advanced && s.advanced.network_key) {
+        s.advanced.network_key = interpetValue(s.advanced.network_key);
+    }
+
+    if (s.frontend && s.frontend.auth_token) {
+        s.frontend.auth_token = interpetValue(s.frontend.auth_token);
+    }
+
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type) => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            const files = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
+
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
+
+    return s;
 }
 
-function addDevice(id) {
+function applyEnvironmentVariables(settings) {
+    const iterate = (obj, path) => {
+        Object.keys(obj).forEach((key) => {
+            if (key !== 'type') {
+                if (key !== 'properties') {
+                    const type = (obj[key].type || 'object').toString();
+                    const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
+                    const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
+                    if (process.env[envVariableName]) {
+                        const setting = path.reduce((acc, val, index) => {
+                            acc[val] = acc[val] || {};
+                            return acc[val];
+                        }, settings);
+
+                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
+                            setting[key] = JSON.parse(process.env[envVariableName]);
+                        } else if (type.indexOf('number') >= 0) {
+                            setting[key] = process.env[envVariableName] * 1;
+                        } else if (type.indexOf('boolean') >= 0) {
+                            setting[key] = process.env[envVariableName].toLowerCase() === 'true';
+                        } else {
+                            /* istanbul ignore else */
+                            if (type.indexOf('string') >= 0) {
+                                setting[key] = process.env[envVariableName];
+                            }
+                        }
+                    }
+                }
+
+                if (typeof obj[key] === 'object') {
+                    const newPath = [...path];
+                    if (key !== 'properties') {
+                        newPath.push(key);
+                    }
+                    iterate(obj[key], newPath);
+                }
+            }
+        });
+    };
+    iterate(schema.properties, []);
+}
+
+function get() {
+    if (!_settings) {
+        _settings = read();
+        applyEnvironmentVariables(_settings);
+    }
+
+    return _settings;
+}
+
+function getWithDefaults() {
+    if (!_settingsWithDefaults) {
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    }
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    return _settingsWithDefaults;
+}
+
+function set(path, value) {
+    let settings = get();
+
+    for (let i = 0; i < path.length; i++) {
+        const key = path[i];
+        if (i === path.length - 1) {
+            settings[key] = value;
+        } else {
+            if (!settings[key]) {
+                settings[key] = {};
+            }
+
+            settings = settings[key];
+        }
+    }
+
+    write();
+}
+
+function apply(newSettings) {
+    ajvSetting(newSettings);
+    const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
+    if (errors.length) {
+        const error = errors[0];
+        throw new Error(`${error.dataPath.substring(1)} ${error.message}`);
+    }
+
+    get(); // Ensure _settings is intialized.
+    _settings = objectAssignDeep.noMutate(_settings, newSettings);
+    write();
+
+    ajvRestartRequired(newSettings);
+    const restartRequired = ajvRestartRequired.errors &&
+        !!ajvRestartRequired.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
+}
+
+function getGroup(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.groups[IDorName];
+    if (byID) {
+        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, group] of Object.entries(settings.groups)) {
+        if (group.friendly_name === IDorName) {
+            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getGroups() {
+    const settings = getWithDefaults();
+    return Object.entries(settings.groups).map(([ID, group]) => {
+        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+    });
+}
+
+function getGroupThrowIfNotExists(IDorName) {
+    const group = getGroup(IDorName);
+    if (!group) {
+        throw new Error(`Group '${IDorName}' does not exist`);
+    }
+
+    return group;
+}
+
+function getDevice(IDorName) {
+    const settings = getWithDefaults();
+    const byID = settings.devices[IDorName];
+    if (byID) {
+        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, device] of Object.entries(settings.devices)) {
+        if (device.friendly_name === IDorName) {
+            return {...device, ID, friendlyName: device.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getDeviceThrowIfNotExists(IDorName) {
+    const device = getDevice(IDorName);
+    if (!device) {
+        throw new Error(`Device '${IDorName}' does not exist`);
+    }
+
+    return device;
+}
+
+function getEntity(IDorName) {
+    const device = getDevice(IDorName);
+    if (device) {
+        return {...device, type: 'device'};
+    }
+
+    const group = getGroup(IDorName);
+    if (group) {
+        return {...group, type: 'group'};
+    }
+
+    return null;
+}
+
+function addDevice(ID) {
+    if (getDevice(ID)) {
+        throw new Error(`Device '${ID}' already exists`);
+    }
+
+    const settings = get();
+
     if (!settings.devices) {
         settings.devices = {};
     }
 
-    settings.devices[id] = {friendly_name: id, retain: false};
-    writeRead();
+    settings.devices[ID] = {friendly_name: ID};
+    write();
+    return getDevice(ID);
 }
 
-function removeDevice(id) {
-    if (settings.devices && settings.devices[id]) {
-        delete settings.devices[id];
-        writeRead();
+// Legacy: can be removed after bridgeLegacy has been removed
+function whitelistDevice(ID) {
+    const settings = get();
+    if (!settings.whitelist) {
+        settings.whitelist = [];
+    }
+
+    if (settings.whitelist.includes(ID)) {
+        throw new Error(`Device '${ID}' already whitelisted`);
+    }
+
+    settings.whitelist.push(ID);
+    write();
+}
+
+function blockDevice(ID) {
+    const settings = get();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
+function banDevice(ID) {
+    const settings = get();
+    if (!settings.ban) {
+        settings.ban = [];
+    }
+
+    settings.ban.push(ID);
+    write();
+}
+
+function removeDevice(IDorName) {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = get();
+    delete settings.devices[device.ID];
+    write();
+}
+
+function addGroup(name, ID=null) {
+    utils.validateFriendlyName(name, true);
+    if (getGroup(name) || getDevice(name)) {
+        throw new Error(`friendly_name '${name}' is already in use`);
+    }
+
+    const settings = get();
+    if (!settings.groups) {
+        settings.groups = {};
+    }
+
+    if (ID == null) {
+        // look for free ID
+        ID = '1';
+        while (settings.groups.hasOwnProperty(ID)) {
+            ID = (Number.parseInt(ID) + 1).toString();
+        }
+    } else {
+        // ensure provided ID is not in use
+        ID = ID.toString();
+        if (settings.groups.hasOwnProperty(ID)) {
+            throw new Error(`group id '${ID}' is already in use`);
+        }
+    }
+
+    settings.groups[ID] = {friendly_name: name};
+    write();
+
+    return getGroup(ID);
+}
+
+function groupHasDevice(group, keys) {
+    for (const device of group.devices) {
+        const index = keys.indexOf(device);
+        if (index != -1) {
+            return keys[index];
+        }
+    }
+
+    return false;
+}
+
+function addDeviceToGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
+    }
+
+    if (!groupHasDevice(group, keys)) {
+        group.devices.push(keys[0]);
+        write();
     }
 }
 
-function getIDByFriendlyName(friendlyName) {
-    if (!settings.devices) {
-        return null;
+function removeDeviceFromGroup(groupIDorName, keys) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        group.devices = [];
     }
 
-    return Object.keys(settings.devices).find((id) =>
-        settings.devices[id].friendly_name === friendlyName
-    );
+    const key = groupHasDevice(group, keys);
+    if (key) {
+        group.devices = group.devices.filter((d) => d != key);
+        write();
+    }
+}
+
+function removeGroup(groupIDorName) {
+    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
+    const settings = get();
+    delete settings.groups[groupID];
+    write();
+}
+
+function changeEntityOptions(IDorName, newOptions) {
+    const settings = get();
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
+}
+
+function changeFriendlyName(IDorName, newName) {
+    utils.validateFriendlyName(newName, true);
+    if (getGroup(newName) || getDevice(newName)) {
+        throw new Error(`friendly_name '${newName}' is already in use`);
+    }
+
+    const settings = get();
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
 }
 
 module.exports = {
-    get: () => settings,
-    write: () => write(),
-    getDevice: (id) => settings.devices ? settings.devices[id] : null,
-    addDevice: (id) => addDevice(id),
-    removeDevice: (id) => removeDevice(id),
-    getIDByFriendlyName: (friendlyName) => getIDByFriendlyName(friendlyName),
+    validate,
+    get: getWithDefaults,
+    set,
+    apply,
+    getDevice,
+    getGroup,
+    getGroups,
+    getEntity,
+    whitelistDevice,
+    banDevice,
+    blockDevice,
+    addDevice,
+    removeDevice,
+    addGroup,
+    removeGroup,
+    addDeviceToGroup,
+    removeDeviceFromGroup,
+    changeEntityOptions,
+    changeFriendlyName,
+    schema,
+
+    // For tests only
+    _write: write,
+    _reRead: () => {
+        _settings = null;
+        get();
+        _settingsWithDefaults = null;
+        getWithDefaults();
+    },
+    _clear: () => {
+        _settings = null;
+        _settingsWithDefaults = null;
+    },
+    _getDefaults: () => defaults,
 };

lib/util/settings.schema.json

@@ -0,0 +1,740 @@
+{
+  "type": "object",
+  "properties": {
+    "device_options": {
+      "type": "object"
+    },
+    "homeassistant": {
+      "title": "Home Assistant integration",
+      "type": "boolean",
+      "description": "Home Assistant integration (MQTT discovery)",
+      "default": false
+    },
+    "permit_join": {
+      "type": "boolean",
+      "default": false,
+      "title": "Permit join",
+      "description": "Allow new devices to join (re-applied at restart)"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
+    "mqtt": {
+      "type": "object",
+      "title": "MQTT",
+      "properties": {
+        "base_topic": {
+          "type": "string",
+          "title": "Base topic",
+          "requiresRestart": true,
+          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
+          "examples": ["zigbee2mqtt"]
+        },
+        "server": {
+          "type": "string",
+          "title": "MQTT server",
+          "requiresRestart": true,
+          "description": "MQTT server URL (use mqtts:// for SSL/TLS connection)",
+          "examples": ["mqtt://localhost:1883"]
+        },
+        "keepalive": {
+          "type": "number",
+          "title": "Keepalive",
+          "requiresRestart": true,
+          "description": "MQTT keepalive in second",
+          "default": 60
+        },
+        "ca": {
+          "type": "string",
+          "title": "Certificate authority",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS certificate of CA used to sign server and client certificates",
+          "examples": ["/etc/ssl/mqtt-ca.crt"]
+        },
+        "key": {
+          "type": "string",
+          "title": "SSL/TLS key",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS key for client-authentication",
+          "examples": ["/etc/ssl/mqtt-client.key"]
+        },
+        "cert": {
+          "type": "string",
+          "title": "SSL/TLS certificate",
+          "description": "Absolute path to SSL/TLS certificate for client-authentication",
+          "requiresRestart": true,
+          "examples": ["/etc/ssl/mqtt-client.crt"]
+        },
+        "user": {
+          "type": "string",
+          "title": "User",
+          "requiresRestart": true,
+          "description": "MQTT server authentication user",
+          "examples": ["johnnysilverhand"]
+        },
+        "password": {
+          "type": "string",
+          "title": "Password",
+          "requiresRestart": true,
+          "description": "MQTT server authentication password",
+          "examples": ["ILOVEPELMENI"]
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "requiresRestart": true,
+          "description": "MQTT client ID",
+          "examples": ["MY_CLIENT_ID"]
+        },
+        "reject_unauthorized": {
+          "type": "boolean",
+          "title": "Reject unauthorized",
+          "requiresRestart": true,
+          "description": "Disable self-signed SSL certificate",
+          "default": true
+        },
+        "include_device_information": {
+          "type": "boolean",
+          "title": "Include device information",
+          "description": "Include device information to mqtt messages",
+          "default": false
+        },
+        "version": {
+          "type": ["number", "null"],
+          "title": "Version",
+          "requiresRestart": true,
+          "description": "MQTT protocol version",
+          "default": 4,
+          "examples": [4, 5]
+        },
+        "force_disable_retain": {
+          "type": "boolean",
+          "title": "Force disable retain",
+          "requiresRestart": true,
+          "description": "Disable retain for all send messages. ONLY enable if you MQTT broker doesn't support retained message (e.g. AWS IoT core, Azure IoT Hub, Google Cloud IoT core, IBM Watson IoT Platform). Enabling will break the Home Assistant integration",
+          "default": false
+        }
+      },
+      "required": ["base_topic", "server"]
+    },
+    "serial": {
+      "type": "object",
+      "title": "Serial",
+      "properties": {
+        "port": {
+          "type": ["string", "null"],
+          "title": "Port",
+          "requiresRestart": true,
+          "description": "Location of the adapter. To autodetect the port, set null",
+          "examples": ["/dev/ttyACM0"]
+        },
+        "disable_led": {
+          "type": "boolean",
+          "title": "Disable led",
+          "requiresRestart": true,
+          "description": "Disable LED of the adapter if supported",
+          "default": false
+        },
+        "adapter": {
+          "type": ["string", "null"],
+          "enum": ["deconz", "zstack", "zigate"],
+          "title": "Adapter",
+          "requiresRestart": true,
+          "description": "Adapter type, not needed unless you are experiencing problems"
+        }
+      }
+    },
+    "blocklist": {
+      "title": "Blocklist",
+      "requiresRestart": true,
+      "description": "Block devices from the network (by ieeeAddr)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "passlist": {
+      "title": "Passlist",
+      "requiresRestart": true,
+      "description": "Allow only certain devices to join the network (by ieeeAddr). Note that all devices not on the passlist will be removed from the network!",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
+      "properties": {
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [11, 15, 20, 25]
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to supress logging"
+        },
+        "log_directory": {
+          "type": "string",
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
+        },
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baudrate for serial port, default: 115200 for Z-Stack, 38400 for Deconz",
+          "examples": [38400, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "availability_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "default": 0,
+          "requiresRestart": true,
+          "title": "Availability Timeout",
+          "description": "Availability timeout in seconds when enabled, devices will be checked if they are still online. Only AC powered routers are checked for availability"
+        },
+        "availability_blocklist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability Blocklist",
+          "description": "Prevent devices from being checked for availability"
+        },
+        "availability_passlist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability passlist",
+          "description": "Only enable availability check for certain devices"
+        },
+        "availability_blacklist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "title": "Availability blacklist (deprecated, use availability_blocklist)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "availability_whitelist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "items": {
+            "type": "string"
+          },
+          "title": "Availability whitelist (deprecated, use passlist)"
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+          "default": true
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "tcp4",
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/dev/log",
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        }
+      }
+    },
+    "map_options": {
+      "type": "object",
+      "title": "Networkmap",
+      "properties": {
+        "graphviz": {
+          "type": "object",
+          "properties": {
+            "colors": {
+              "type": "object",
+              "properties": {
+                "fill": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "font": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "line": {
+                  "type": "object",
+                  "properties": {
+                    "active": {
+                      "type": "string"
+                    },
+                    "inactive": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "ota": {
+      "type": "object",
+      "title": "OTA updates",
+      "properties": {
+        "update_check_interval": {
+          "type": "number",
+          "title": "Update check interval",
+          "description": "Your device may request a check for a new firmware update. This value determines how frequently third party servers may actually be contacted to look for firmware updates. The value is set in minutes, and the default is 10.",
+          "default": 10
+        },
+        "disable_automatic_update_check": {
+          "type": "boolean",
+          "title": "Disable automatic update check",
+          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
+          "default": false
+        }
+      }
+    },
+    "devices": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^0x[\\d\\w]{16}$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/device"
+        }
+      }
+    },
+    "groups": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[\\w].*$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/group"
+        }
+      }
+    },
+    "frontend": {
+      "type": "object",
+      "title": "Frontend",
+      "properties": {
+        "port": {
+          "type": "number",
+          "title": "Port",
+          "description": "Frontend binding port",
+          "default": 8080,
+          "requiresRestart": true
+        },
+        "host": {
+          "type": "string",
+          "title": "Bind host",
+          "description": "Frontend binding host",
+          "default":" 0.0.0.0",
+          "requiresRestart": true
+        },
+        "auth_token": {
+          "type": ["string", "null"],
+          "title": "Auth token",
+          "description": "Enables authentication, disabled by default",
+          "requiresRestart": true
+        }
+      }
+    }
+  },
+  "required": ["mqtt"],
+  "definitions": {
+    "device": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string",
+          "title": "Friendly name",
+          "description": "Used in the MQTT topic of a device. By default this is the device ID",
+          "readOnly": true
+        },
+        "retain": {
+          "type": "boolean",
+          "title": "Retain",
+          "description": "Retain MQTT messages of this device"
+        },
+        "retention": {
+          "type": "number",
+          "title": "Retention",
+          "description": "Sets the MQTT Message Expiry in seconds, Make sure to set mqtt.version to 5"
+        },
+        "qos": {
+          "type": "number",
+          "title": "QoS",
+          "descritption": "QoS level for MQTT messages of this device"
+        },
+        "debounce": {
+          "type": "number",
+          "title": "Debounce",
+          "description": "Debounces messages of this device"
+        },
+        "debounce_ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["action"],
+          "title": "Ignore debounce",
+          "description": "Protects unique payload values of specified payload properties from overriding within debounce time"
+        },
+        "optimistic": {
+          "type": "boolean",
+          "title": "Optimistic",
+          "description": "Publish optimistic state after set (default true)"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["temperature", "battery", "action"],
+          "title": "Filtered attributes",
+          "description": "Allows to prevent certain attributes from being published"
+        }
+      },
+      "required": ["friendly_name"]
+    },
+    "group": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string"
+        },
+        "retain": {
+          "type": "boolean"
+        },
+        "devices": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "optimistic": {
+          "type": "boolean"
+        },
+        "qos": {
+          "type": "number"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["friendly_name"]
+    }
+  }
+}

lib/util/utils.js

@@ -0,0 +1,254 @@
+const equals = require('fast-deep-equal/es6');
+const humanizeDuration = require('humanize-duration');
+const data = require('./data');
+const vm = require('vm');
+const fs = require('fs');
+const path = require('path');
+
+// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
+const xiaomiManufacturerID = [4151, 4447];
+const ikeaTradfriManufacturerID = [4476];
+
+// construct a local ISO8601 string (instead of UTC-based)
+// Example:
+//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
+//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
+function toLocalISOString(dDate) {
+    const tzOffset = -dDate.getTimezoneOffset();
+    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
+    const pad = function(num) {
+        const norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+    };
+
+    return dDate.getFullYear() +
+        '-' + pad(dDate.getMonth() + 1) +
+        '-' + pad(dDate.getDate()) +
+        'T' + pad(dDate.getHours()) +
+        ':' + pad(dDate.getMinutes()) +
+        ':' + pad(dDate.getSeconds()) +
+        plusOrMinus + pad(tzOffset / 60) +
+        ':' + pad(tzOffset % 60);
+}
+
+const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool', 'water', 'meter',
+];
+
+function capitalize(s) {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
+async function getZigbee2mqttVersion() {
+    return new Promise((resolve, reject) => {
+        const git = require('git-last-commit');
+        const packageJSON = require('../../package.json');
+        const version = packageJSON.version;
+
+        git.getLastCommit((err, commit) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = require('../../.hash.json').hash;
+                } catch (error) {
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
+            }
+
+            resolve({commitHash, version});
+        });
+    });
+}
+
+async function getDependencyVersion(depend) {
+    return new Promise((resolve, reject) => {
+        const packageJSON = require('../../node_modules/'+depend+'/package.json');
+        const version = packageJSON.version;
+        resolve({version});
+    });
+}
+
+function formatDate(date, type) {
+    let result;
+
+    switch (type) {
+    case 'ISO_8601':
+        result = new Date(date).toISOString();
+        break;
+    case 'ISO_8601_local':
+        result = toLocalISOString(new Date(date));
+        break;
+    case 'epoch':
+        result = date;
+        break;
+    case 'relative':
+        // https://github.com/EvanHahn/HumanizeDuration.js#options
+        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
+        break;
+    default:
+        throw new Error(`Unsupported type '${type}'`);
+    }
+
+    return result;
+}
+
+function objectHasProperties(object, properties) {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function equalsPartial(object, expected) {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function getObjectProperty(object, key, defaultValue) {
+    return object.hasOwnProperty(key) ? object[key] : defaultValue;
+}
+
+function getResponse(request, data, error) {
+    const response = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+function parseJSON(value, failedReturnValue) {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+function loadModuleFromText(moduleCode) {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    return sandbox.module.exports;
+}
+
+function loadModuleFromFile(modulePath) {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
+function* getExternalConvertersDefinitions(settings) {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converter;
+
+        if (moduleName.endsWith('.js')) {
+            converter = loadModuleFromFile(data.joinPath(moduleName));
+        } else {
+            converter = require(moduleName);
+        }
+
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+function toSnakeCase(value) {
+    if (typeof value === 'object') {
+        value = {...value};
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
+function validateFriendlyName(name, throwFirstError=false) {
+    const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
+    return errors;
+}
+function sleep(seconds) {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+module.exports = {
+    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
+    secondsToMilliseconds: (seconds) => seconds * 1000,
+    getZigbee2mqttVersion,
+    getDependencyVersion,
+    objectHasProperties,
+    toSnakeCase,
+    sleep,
+    getObjectProperty,
+    getEndpointNames: () => endpointNames,
+    isXiaomiDevice: (device) => {
+        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+    },
+    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
+    formatDate: (date, type) => formatDate(date, type),
+    equalsPartial,
+    getResponse,
+    capitalize,
+    parseJSON,
+    getExternalConvertersDefinitions,
+    validateFriendlyName,
+    loadModuleFromFile,
+    loadModuleFromText,
+};

lib/util/yaml.js

@@ -0,0 +1,36 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+const equals = require('fast-deep-equal/es6');
+
+function read(file) {
+    try {
+        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+function readIfExists(file, default_=null) {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+function writeIfChanged(file, content) {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.safeDump(content));
+    }
+}
+
+function updateIfChanged(file, key, value) {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/zigbee.js

@@ -1,219 +1,323 @@
-const ZShepherd = require('zigbee-shepherd');
+const ZigbeeHerdsman = require('zigbee-herdsman');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
 const data = require('./util/data');
-const zclPacket = require('zcl-packet');
+const assert = require('assert');
+const utils = require('./util/utils');
+const events = require('events');
+const objectAssignDeep = require('object-assign-deep');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify-without-jsonify');
 
-const advancedSettings = settings.get().advanced;
+const endpointNames = utils.getEndpointNames();
+const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
-const shepherdSettings = {
-    net: {
-        panId: advancedSettings && advancedSettings.pan_id ? advancedSettings.pan_id : 0x1a62,
-        channelList: [advancedSettings && advancedSettings.channel ? advancedSettings.channel : 11],
-    },
-    dbPath: data.joinPath('database.db'),
-    sp: {
-        baudrate: advancedSettings && advancedSettings.baudrate ? advancedSettings.baudrate : 115200,
-        rtscts: advancedSettings && (typeof(advancedSettings.rtscts) === 'boolean') ? advancedSettings.rtscts : true,
-    },
-};
-
-logger.debug(`Using zigbee-shepherd with settings: '${JSON.stringify(shepherdSettings)}'`);
-
-class Zigbee {
+class Zigbee extends events.EventEmitter {
     constructor() {
-        this.handleReady = this.handleReady.bind(this);
-        this.handleMessage = this.handleMessage.bind(this);
-        this.handleError = this.handleError.bind(this);
+        super();
+        this.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler.bind(this);
     }
 
-    start(onMessage, callback) {
-        logger.info(`Starting zigbee-shepherd`);
-
-        this.shepherd = new ZShepherd(settings.get().serial.port, shepherdSettings);
-
-        this.shepherd.start((error) => {
-            if (error) {
-                logger.info('Error while starting zigbee-shepherd, attemping to fix... (takes 60 seconds)');
-                this.shepherd.controller._znp.close((() => null));
-
-                setTimeout(() => {
-                    logger.info(`Starting zigbee-shepherd`);
-                    this.shepherd.start((error) => {
-                        if (error) {
-                            logger.error('Error while starting zigbee-shepherd!');
-                            logger.error(
-                                'Press the reset button on the stick (the one closest to the USB) and start again'
-                            );
-                            callback(error);
-                        } else {
-                            logger.info('zigbee-shepherd started');
-                            callback(null);
-                        }
-                    });
-                }, 60 * 1000);
-            } else {
-                logger.info('zigbee-shepherd started');
-                callback(null);
-            }
-        });
-
-        // Register callbacks.
-        this.shepherd.on('ready', this.handleReady);
-        this.shepherd.on('ind', this.handleMessage);
-        this.shepherd.on('error', this.handleError);
-
-        this.onMessage = onMessage;
-    }
-
-    softReset(callback) {
-        this.shepherd.reset('soft', callback);
-    }
-
-    stop(callback) {
-        this.shepherd.stop((error) => {
-            logger.info('zigbee-shepherd stopped');
-            callback(error);
-        });
-    }
-
-    handleReady() {
-        // Set all Xiaomi devices (manufId === 4151) to be online, so shepherd won't try
-        // to query info from devices (which would fail because they go tosleep).
-        // Xiaomi lumi.plug has manufId === 4447 and can be in the sleep mode too
-        const devices = this.getAllClients();
-        devices.forEach((d) => {
-            if ((d.manufId === 4151) || (d.manufId === 4447)) {
-                const device = this.shepherd.find(d.ieeeAddr, 1);
-                if (device) {
-                    device.getDevice().update({
-                        status: 'online',
-                        joinTime: Math.floor(Date.now() / 1000),
-                    });
-                }
-            }
-        });
-
-        logger.info('zigbee-shepherd ready');
-    }
-
-    handleError(message) {
-        // This event may appear if zigbee-shepherd cannot decode bad packets (invalid checksum).
-        logger.error(message);
-    }
-
-    permitJoin(permit) {
-        if (permit) {
-            logger.info('Zigbee: allowing new devices to join.');
-        } else {
-            logger.info('Zigbee: disabling joining new devices.');
-        }
-
-        this.shepherd.permitJoin(permit ? 255 : 0, (error) => {
-            if (error) {
-                logger.info(error);
-            }
-        });
-    }
-
-    getAllClients() {
-        return this.shepherd.list().filter((device) => device.type !== 'Coordinator');
-    }
-
-    removeDevice(deviceID, callback) {
-        this.shepherd.remove(deviceID, (error) => {
-            if (error) {
-                logger.warn(`Failed to remove '${deviceID}', trying force remove...`);
-                this._forceRemove(deviceID, callback);
-            } else {
-                callback(null);
-            }
-        });
-    }
-
-    _forceRemove(deviceID, callback) {
-        const device = this.shepherd._findDevByAddr(deviceID);
-
-        if (device) {
-            return this.shepherd._unregisterDev(device, (error) => callback(error));
-        } else {
-            logger.warn(`Could not find ${deviceID} for force removal`);
-            callback(true);
-        }
-    }
-
-    ping(deviceID) {
-        const device = this.shepherd._findDevByAddr(deviceID);
-
-        if (device) {
-            // Note: checkOnline has the callback argument but does not call callback
-            logger.debug(`Check online ${deviceID}`);
-            this.shepherd.controller.checkOnline(device);
-        }
-    }
-
-    handleMessage(message) {
-        if (this.onMessage) {
-            this.onMessage(message);
-        }
-    }
-
-    getDevice(deviceID) {
-        return this.shepherd.list().find((d) => d.ieeeAddr === deviceID);
-    }
-
-    getCoordinator() {
-        const device = this.shepherd.list().find((d) => d.type === 'Coordinator');
-        return this.shepherd.find(device.ieeeAddr, 1);
-    }
-
-    publish(deviceID, cid, cmd, zclData, ep, callback) {
-        const device = this._findDevice(deviceID, ep);
-        if (!device) {
-            logger.error(`Zigbee cannot publish message to device because '${deviceID}' not known by zigbee-shepherd`);
-            return;
-        }
-
-        logger.info(`Zigbee publish to '${deviceID}', ${cid} - ${cmd} - ${JSON.stringify(zclData)} - ${ep}`);
-        device.functional(cid, cmd, zclData, callback);
-    }
-
-    read(deviceID, cid, attr, ep, callback) {
-        const device = this._findDevice(deviceID, ep);
-        if (!device) {
-            logger.error(`Zigbee cannot read attribute from device because '${deviceID}' not known by zigbee-shepherd`);
-            return;
-        }
-
-        device.read(cid, attr, callback);
-    }
-
-    registerOnAfIncomingMsg(ieeeAddr, ep) {
-        const device = this._findDevice(ieeeAddr, ep);
-        device.onAfIncomingMsg = (message) => {
-            // Parse the message
-            zclPacket.parse(message.data, message.clusterid, (error, zclData) => {
-                const message = {
-                    endpoints: [device],
-                    data: zclData,
-                };
-
-                this.handleMessage(message);
-            });
+    async start() {
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key,
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
+            },
         };
-    }
 
-    _findDevice(deviceID, ep) {
-        // Find device in zigbee-shepherd
-        let device = this.getDevice(deviceID);
-        if (!device || !device.epList || !device.epList.length) {
-            logger.error(`Zigbee cannot determine endpoint for '${deviceID}'`);
-            return null;
+        const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
+        herdsmanSettingsLog.network.networkKey = 'HIDDEN';
+        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
+
+        if (herdsmanSettings.network.networkKey === 'GENERATE') {
+            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+            settings.set(['advanced', 'network_key'], newKey);
+            herdsmanSettings.network.networkKey = newKey;
         }
 
-        ep = ep ? ep : device.epList[0];
-        device = this.shepherd.find(deviceID, ep);
-        return device;
+        if (herdsmanSettings.network.panID === 'GENERATE') {
+            const newPanID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
+            settings.set(['advanced', 'pan_id'], newPanID);
+            herdsmanSettings.network.panID = newPanID;
+        }
+
+        try {
+            herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
+            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
+            await this.herdsman.start();
+        } catch (error) {
+            logger.error(`Error while starting zigbee-herdsman`);
+            throw error;
+        }
+
+        this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
+        this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
+        this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
+        this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
+        this.herdsman.on('message', (data) => this.emit('event', 'message', data));
+        this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
+
+        logger.info('zigbee-herdsman started');
+        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
+
+        for (const device of this.getClients()) {
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
+                    device.removeFromNetwork();
+                }
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
+                device.removeFromNetwork();
+            }
+        }
+
+        // Check if we have to turn off the led
+        if (settings.get().serial.disable_led) {
+            await this.herdsman.setLED(false);
+        }
+
+        // Check if we have to set a transmit power
+        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().experimental.transmit_power;
+            await this.herdsman.setTransmitPower(transmitPower);
+            logger.info(`Set transmit power to '${transmitPower}'`);
+        }
+    }
+
+    async getCoordinatorVersion() {
+        return this.herdsman.getCoordinatorVersion();
+    }
+
+    isStopping() {
+        return this.herdsman.isStopping();
+    }
+
+    async getNetworkParameters() {
+        return this.herdsman.getNetworkParameters();
+    }
+
+    async reset(type) {
+        await this.herdsman.reset(type);
+    }
+
+    async stop() {
+        await this.herdsman.stop();
+        logger.info('zigbee-herdsman stopped');
+    }
+
+    async permitJoin(permit, resolvedEntity, time=undefined) {
+        permit ?
+            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
+            logger.info('Zigbee: disabling joining new devices.');
+
+        if (resolvedEntity && permit) {
+            await this.herdsman.permitJoin(permit, resolvedEntity.device, time);
+        } else {
+            await this.herdsman.permitJoin(permit, undefined, time);
+        }
+    }
+
+    async getPermitJoin() {
+        return this.herdsman.getPermitJoin();
+    }
+
+    getClients() {
+        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
+    }
+
+    getDevices() {
+        return this.herdsman.getDevices();
+    }
+
+    getDeviceByIeeeAddr(ieeeAddr) {
+        return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
+    }
+
+    getDeviceByNetworkAddress(networkAddress) {
+        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
+    }
+
+    getDevicesByType(type) {
+        return this.herdsman.getDevicesByType(type);
+    }
+
+    /**
+     * @param {string} key
+     * @return {object} {
+     *      type: device | coordinator
+     *      device|group: zigbee-herdsman entity
+     *      endpoint: selected endpoint (only if type === device)
+     *      settings: from configuration.yaml
+     *      name: name of the entity
+     *      definition: zigbee-herdsman-converters definition (only if type === device)
+     * }
+     */
+    resolveEntity(key) {
+        assert(
+            typeof key === 'string' || typeof key === 'number' ||
+            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            `Wrong type '${typeof key}'`,
+        );
+
+        const deviceOptions = settings.get().device_options;
+        if (typeof key === 'string' || typeof key === 'number') {
+            if (typeof key === 'number') {
+                key = key.toString();
+            }
+
+            if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
+                const coordinator = this.getDevicesByType('Coordinator')[0];
+                return {
+                    type: 'device',
+                    device: coordinator,
+                    endpoint: coordinator.getEndpoint(1),
+                    settings: {friendlyName: 'Coordinator'},
+                    name: 'Coordinator',
+                };
+            }
+
+            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            const endpointByNumber = key.match(keyEndpointByNumber);
+            if (!endpointKey && endpointByNumber) {
+                endpointKey = Number(endpointByNumber[1]);
+            }
+            if (endpointKey) {
+                key = key.replace(`/${endpointKey}`, '');
+            }
+
+            const entity = settings.getEntity(key);
+            if (!entity) {
+                return null;
+            } else if (entity.type === 'device') {
+                const device = this.getDeviceByIeeeAddr(entity.ID);
+                if (!device) {
+                    return null;
+                }
+
+                const definition = zigbeeHerdsmanConverters.findByDevice(device);
+                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
+                let endpoint;
+                if (endpointKey) {
+                    if (endpointByNumber) {
+                        endpoint = device.getEndpoint(endpointKey);
+                    } else {
+                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
+                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpoints[endpointKey];
+                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
+                        endpoint = device.getEndpoint(endpointID);
+                    }
+                } else if (endpoints && endpoints['default']) {
+                    endpoint = device.getEndpoint(endpoints['default']);
+                } else {
+                    endpoint = device.endpoints[0];
+                }
+
+                return {
+                    type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
+                    name: entity.friendlyName, definition,
+                };
+            } else {
+                let group = this.getGroupByID(entity.ID);
+                if (!group) group = this.createGroup(entity.ID);
+                return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
+            }
+        } else if (key.constructor.name === 'Device') {
+            const setting = settings.getEntity(key.ieeeAddr);
+            return {
+                type: 'device',
+                device: key,
+                endpoint: key.endpoints[0],
+                settings: {...deviceOptions, ...(setting || {})},
+                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
+                definition: zigbeeHerdsmanConverters.findByDevice(key),
+            };
+        } else { // Group
+            const setting = settings.getEntity(key.groupID);
+            return {
+                type: 'group',
+                group: key,
+                settings: {...deviceOptions, ...(setting || {})},
+                name: setting ? setting.friendlyName : key.groupID,
+            };
+        }
+    }
+
+    getGroupByID(ID) {
+        return this.herdsman.getGroupByID(ID);
+    }
+
+    getGroups() {
+        return this.herdsman.getGroups();
+    }
+
+    createGroup(groupID) {
+        return this.herdsman.createGroup(groupID);
+    }
+
+    acceptJoiningDeviceHandler(ieeeAddr) {
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
+                return true;
+            } else {
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
+                return false;
+            }
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
+                return false;
+            } else {
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    async touchlinkFactoryResetFirst() {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr, channel) {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr, channel) {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan() {
+        return this.herdsman.touchlinkScan();
     }
 }
 

package.json

@@ -1,12 +1,15 @@
 {
   "name": "zigbee2mqtt",
-  "version": "0.1.0",
-  "description": "Zigbee to MQTT bridge using zigbee-shepherd",
+  "version": "1.18.0",
+  "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
+  "engines": {
+    "node": "^10 || ^12 || ^14 || ^15"
+  },
   "keywords": [
     "xiaomi",
     "tradfri",
@@ -17,30 +20,63 @@
     "cc2531"
   ],
   "scripts": {
+    "test-with-coverage": "jest test --coverage",
+    "eslint": "node_modules/.bin/eslint lib/",
     "start": "node index.js",
-    "docgen": "node support/docgen.js",
-    "verify-homeassistant-mapping": "node support/verify-homeassistant-mapping.js",
-    "eslint": "node_modules/.bin/eslint ."
+    "test": "jest test",
+    "test-watch": "jest test --watch"
   },
   "author": "Koen Kanters",
   "license": "GPL-3.0",
   "bugs": {
     "url": "https://github.com/Koenkk/zigbee2mqtt/issues"
   },
-  "homepage": "https://github.com/Koenkk/zigbee2mqtt/wiki",
+  "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "object-assign-deep": "*",
-    "mqtt": "*",
-    "winston": "2.4.2",
-    "zigbee-shepherd": "git+https://github.com/Koenkk/zigbee-shepherd.git",
-    "zigbee-shepherd-converters": "*",
-    "json2yaml": "*",
-    "zcl-packet": "git+https://github.com/Koenkk/zcl-packet.git",
-    "js-yaml": "*",
-    "git-last-commit": "*"
+    "ajv": "^6.12.4",
+    "debounce": "^1.2.0",
+    "deep-object-diff": "^1.1.0",
+    "fast-deep-equal": "^3.1.3",
+    "finalhandler": "^1.1.2",
+    "git-last-commit": "^1.0.0",
+    "humanize-duration": "^3.23.1",
+    "js-yaml": "^3.14.0",
+    "json-stable-stringify-without-jsonify": "=1.0.1",
+    "mkdir-recursive": "^0.4.0",
+    "moment": "^2.27.0",
+    "mqtt": "4.2.1",
+    "object-assign-deep": "^0.4.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.2",
+    "serve-static": "^1.14.1",
+    "winston": "^3.3.3",
+    "winston-syslog": "^2.4.4",
+    "ws": "^7.3.1",
+    "zigbee-herdsman": "0.13.72",
+    "zigbee-herdsman-converters": "14.0.74",
+    "zigbee2mqtt-frontend": "0.3.76"
   },
   "devDependencies": {
     "eslint": "*",
-    "eslint-config-google": "*"
+    "eslint-config-google": "*",
+    "eslint-plugin-jest": "*",
+    "jest": "*",
+    "tmp": "*"
+  },
+  "jest": {
+    "coverageThreshold": {
+      "global": {
+        "branches": 100,
+        "functions": 100,
+        "lines": 100,
+        "statements": 100
+      }
+    },
+    "collectCoverageFrom": [
+      "lib/**/*.js"
+    ]
+  },
+  "bin": {
+    "zigbee2mqtt": "cli.js"
   }
 }

scripts/dumpHomeAssistantMapping.js

@@ -0,0 +1,35 @@
+const HomeAssistant = require('../lib/extension/homeassistant');
+const stringify = require('json-stable-stringify-without-jsonify');
+const homeassistant = new HomeAssistant(null, null, null, null, {on: () => {}});
+const mapping = Object.entries(homeassistant._getMapping());
+mapping.sort((a, b) => {
+    if (a[0] > b[0]) {
+        return -1;
+    }
+    if (b[0] > a[0]) {
+        return 1;
+    }
+    return 0;
+});
+
+for (const map of mapping) {
+    map[1].sort((a, b) => {
+        return JSON.stringify(a).localeCompare(JSON.stringify(b));
+    });
+
+    const toDelete = [];
+    for (const entry of map[1]) {
+        if (entry.type === 'light') {
+            if (entry.discovery_payload.brightness === false) delete entry.discovery_payload.brightness;
+            if (entry.discovery_payload.color_temp === false) delete entry.discovery_payload.color_temp;
+            if (entry.discovery_payload.xy === false) delete entry.discovery_payload.xy;
+            if (entry.discovery_payload.hs === false) delete entry.discovery_payload.hs;
+        } else if (entry.object_id === 'linkquality') {
+            toDelete.push(entry);
+        }
+    }
+
+    toDelete.forEach((d) => map[1].splice(map[1].indexOf(d), 1));
+
+    console.log(`${map[0]} - ${stringify(map[1])}`);
+}

scripts/generateNewSupportedDevices.js

@@ -0,0 +1,10 @@
+const fs = require('fs');
+const filename = process.argv[2];
+const text = fs.readFileSync(filename, 'utf8');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+
+
+for (const line of text.split('\n')) {
+    const model = zigbeeHerdsmanConverters.devices.find((d) => d.model === line);
+    console.log(`- \`${line}\` ${model.vendor} ${model.description}`);
+}

scripts/install.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+function zigbee2mqtt-show-short-info {
+  echo "Setup for Zigbee2MQTT bridge."
+}
+
+function zigbee2mqtt-show-long-info {
+  echo "This script installs the Zigbee2MQTT bridge"
+}
+
+function zigbee2mqtt-show-copyright-info {
+  echo "Original concept by Landrash <https://github.com/landrash>."
+}
+
+function zigbee2mqtt-install-package {
+echo -n "Installing dependencies : "
+node=$(which npm)
+if [ -z "${node}" ]; then #Installing NodeJS if not already installed.
+  printf "Downloading and installing NodeJS...\\n"
+  curl -sL https://deb.nodesource.com/setup_10.x | bash -
+  apt install -y nodejs
+fi
+
+echo "Cloning Zigbee2MQTT git repository"
+git clone https://github.com/Koenkk/zigbee2mqtt.git /opt/zigbee2mqtt
+chown -R pi:pi /opt/zigbee2mqtt
+
+echo "Running install. This might take a while and can produce som expected errors"
+cd /opt/zigbee2mqtt || exit
+su pi -c "npm ci"
+
+echo "Creating service file zigbee2mqtt.service"
+service_path="/etc/systemd/system/zigbee2mqtt.service"
+
+echo "[Unit]
+Description=zigbee2mqtt
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/npm start
+WorkingDirectory=/opt/zigbee2mqtt
+StandardOutput=inherit
+StandardError=inherit
+Restart=always
+User=pi
+
+[Install]
+WantedBy=multi-user.target" > $service_path
+
+echo "Checking the installation..."
+if [ ! -f /opt/zigbee2mqtt/data/configuration.yaml ]; then
+  validation=""
+else
+  validation="ok"
+fi
+
+if [ ! -z "${validation}" ]; then
+  echo
+  echo -e "\\e[32mInstallation done..\\e[0m"
+  echo -e "Update of configuration.yaml is required found at /opt/zigbee2mqtt/data/"
+  echo -e "Some further configuration is required and details can be found here https://www.zigbee2mqtt.io"
+  echo
+  echo -e "Service can be started after configuration but running sudo systemctl start zigbee2mqtt"
+  echo
+else
+  echo
+  echo -e "\\e[31mInstallation failed..."
+  echo
+  return 1
+fi
+return 0
+}
+
+[[ "$_" == "$0" ]] && zigbee2mqtt-install-package

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,89 @@
+const {ZnpCommandStatus, NvSystemIds} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        let deletedCount = 0;
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            let len;
+            const needOsal = !(this.version.product == ZnpVersion.zStack3x0 && id <= 7);
+            if (needOsal) {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength', {id: id});
+                len = lengthRes.payload['length'];
+            } else {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'nvLength',
+                    {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0});
+                len = lengthRes.payload['len'];
+            }
+            if (len != 0) {
+                console.log(`NVMEM item #${id} - deleting, size: ${len}`);
+                if (needOsal) {
+                    await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                        {id: id, len: len},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                } else {
+                    await this.znp.request(Subsystem.SYS, 'nvDelete',
+                        {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                }
+                deletedCount++;
+            }
+        }
+        console.log(`Clearing all NVMEM items finished, deleted ${deletedCount} items`);
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

scripts/zigbee2socat_installer.sh

@@ -0,0 +1,181 @@
+#!/bin/bash
+
+######################################
+##  INITIALIZE                      ##
+######################################
+
+set -o errexit -o pipefail -o noclobber -o nounset
+
+! getopt --test > /dev/null
+if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
+    echo "Unfortunately getopt failed in this environment."
+    exit 2
+fi
+
+
+PROGRAM_NAME="zigbee2socat_installer"
+VERSION="1.0"
+
+
+######################################
+##  PARSE ARGUMENTS                 ##
+######################################
+
+OPTS="a:p:uvmhV"
+LONG="addr:,port:,uninstall,verbose,man,help,version"
+
+# Concern: possible to escape getopt and execute commands as root?
+! PARSED=$(getopt -n $PROGRAM_NAME \
+                  -o $OPTS \
+                  -l $LONG \
+                  -- "$@")
+if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
+    printf "Error parsing arguments. Try %s --help\n" "$PROGRAM_NAME"
+    exit 3
+fi
+eval set -- "$PARSED" # Use remaining arguments that weren't parsed
+
+
+function handle_opts {
+    if [[ $# = 1 ]]; then
+        usage
+    fi
+    while true; do
+    case $1 in
+        -a|--addr)
+            ip=$2; shift 2; continue ;;
+
+        -p|--port)
+            port=$2; shift 2; continue ;;
+
+        -u|--uninstall)
+            uninstall ;;
+
+        -v|--verbose)
+            verbose=1; shift; continue ;;
+        
+        -m|--man)
+            makemanual ;;
+
+        -h|--help)
+            usage ;;
+
+        -V|--version)
+            version ;;
+
+        --) # No more arguments to parse
+            shift; break ;;
+
+        *)
+            printf "Programming error! Option: %s\n" "$1"
+            exit 4 ;;
+    esac;done
+}
+
+
+######################################
+##  START OF MAIN                   ##
+######################################
+
+verbose=0
+ip=""
+port=""
+
+function main {
+    handle_opts "$@"
+
+    # TODO: Check IP-address valid?
+    if [[ "$ip" = "" ]]; then
+        echo "IP-address not specified"
+        exit 5
+    fi
+
+    # Is the port valid?
+    if [[ $port -lt 1 || $port -gt 65535 ]]; then
+        printf "Port %s, is outside range: [1-65535]\n" "$port"
+        exit 6
+    fi
+
+    zigbee-socatvusb-install-package
+}
+
+
+######################################
+##  FUNCTIONS BELOW                 ##
+######################################
+
+function zigbee-socatvusb-install-package {
+    echo "Installing socat:"
+    sudo apt-get install socat
+    echo
+
+    echo "Make dir for zigbee vusb"
+    sudo mkdir -p /opt/zigbee2mqtt/vusb/ || die "Couldn't mkdir /opt/zigbee2mqtt/vusb/"
+    sudo chown -R pi:pi /opt/zigbee2mqtt/vusb/ || die "Couldn't chown /opt/zigbee2mqtt/vusb/"
+
+    echo "Creating service file zigbee-socatvusb.service"
+    service_path="/etc/systemd/system/zigbee-socatvusb.service"
+
+    [[ -f $service_path ]] && sudo rm $service_path
+    echo "[Unit]
+    Description=socat-vusb
+    After=network-online.target
+
+    [Service]
+    User=pi
+    ExecStart=/usr/bin/socat -d -d pty,raw,echo=0,link=/opt/zigbee2mqtt/vusb/zigbee_cc2530 tcp:$ip:$port,reuseaddr
+    Restart=always
+    RestartSec=10
+
+    [Install]
+    WantedBy=multi-user.target" > $service_path || die "Couldn't create service /etc/systemd/system/zigbee-socatvusb.service"
+
+    sudo systemctl --system daemon-reload
+
+    echo "Installation is now complete"
+    echo
+    echo "Service can be started after configuration by running: sudo systemctl start zigbee-socatvusb"
+}
+
+function uninstall {
+    service_path="systemctl status socat-vusb.service"
+    [[ -f $service_path ]] && sudo rm $service_path
+    sudo systemctl --system daemon-reload
+    echo "Uninstalled successfully"
+    exit 0
+}
+
+function makemanual {
+    [[ -f "$PROGRAM_NAME.man" ]] && sudo rm $PROGRAM_NAME.man
+    help2man -N ./$PROGRAM_NAME.sh > $PROGRAM_NAME.man ; man ./$PROGRAM_NAME.man
+    exit 0
+}
+
+function usage {
+    echo -e "\
+    \rSetup for development version of Zigbee2Socat
+
+    -a, --addr IP       Listen on this IP-address
+    -p, --port PORT     Listen on this port
+    -u, --uninstall     Uninstall zigbee-socatvusb.service
+    -v, --verbose       Print more information
+    -m, --man           Make and display manual
+    -h, --help          Display this help message
+    
+    \rOriginal concept by JFLN\
+    "
+    exit 0
+}
+
+function version {
+    echo "$PROGRAM_NAME $VERSION"
+    exit 0
+}
+
+function die {
+    printf "%s\n" "$1"
+    exit 1
+}
+
+
+main "$@" # Call main-function last

support/docgen.js

@@ -1,127 +0,0 @@
-/**
- * This script generates the supported devices page.
- * Run by executing: npm run docs
- */
-
-const deviceMapping = require('zigbee-shepherd-converters').devices;
-const fs = require('fs');
-const YAML = require('json2yaml');
-const homeassistant = require('../lib/homeassistant');
-
-const outputdir = process.argv[2];
-
-if (!outputdir) {
-    console.error("Please specify an output directory");
-}
-
-let file = 'Supported-devices.md';
-let text = '*NOTE: Automatically generated by `npm run docgen`*\n';
-text += `
-In case you own a Zigbee device which is **NOT** listed here, please see [How to support new devices](https://github.com/Koenkk/zigbee2mqtt/wiki/How-to-support-new-devices).
-\n`;
-
-const logDevices = (devices) => {
-    let result = '';
-    result += '| Model | Description | Picture |\n';
-    result += '| ------------- | ------------- | -------------------------- |\n';
-    devices = new Map(devices.map((d) => [d.model, d]));
-    devices.forEach((device) => {
-        const pathFriendly = device.model.replace(new RegExp("/", "g"), '-').replace(new RegExp(":", "g"), '-').replace(new RegExp(" ", "g"), '-');
-        result += `| ${device.model} | ${device.vendor} ${device.description} (${device.supports}) | ![${pathFriendly}](images/devices/${pathFriendly}.jpg) |\n`;
-    });
-
-    return result;
-}
-
-const vendors = Array.from(new Set(deviceMapping.map((d) => d.vendor)));
-vendors.sort();
-vendors.forEach((vendor) => {
-    text += `### ${vendor}\n`;
-    text += logDevices(deviceMapping.filter((d) => d.vendor === vendor));
-    text += '\n';
-})
-
-fs.writeFileSync(outputdir + '/' + file, text);
-
-
-file = 'Integrating-with-Home-Assistant.md';
-text = '*NOTE: Automatically generated by `npm run docgen`*\n\n';
-text += 'If you\'re hosting zigbee2mqtt using [this hassio addon-on](https://github.com/danielwelch/hassio-zigbee2mqtt) use their documentation on how to configure.\n\n'
-text += 'The easiest way to integrate zigbee2mqtt with Home Assistant is by using [MQTT discovery](https://www.home-assistant.io/docs/mqtt/discovery/).\n\n'
-text += 'To achieve the best possible integration (including MQTT discovery):\n';
-text += '- In your **zigbee2mqtt** `configuration.yaml` set `homeassistant: true`\n'
-text += '- In your **Home Assistant** `configuration.yaml`:\n';
-
-text += '```yaml\n';
-text += 'mqtt:\n';
-text += '  discovery: true\n';
-text += '  broker: [YOUR MQTT BROKER]  # Remove if you want to use builtin-in MQTT broker\n';
-text += '  birth_message:\n';
-text += `    topic: 'hass/status'\n`;
-text += `    payload: 'online'\n`;
-text += '  will_message:\n';
-text += `    topic: 'hass/status'\n`;
-text += `    payload: 'offline'\n`;
-text += '```';
-
-text += '\n\n'
-
-text += 'To respond to button clicks (e.g. WXKG01LM) you can use the following Home Assistant configuration:\n'
-text += '```yaml'
-text += `
-automation:
-  - alias: Respond to button clicks
-    trigger:
-      platform: mqtt
-      topic: 'zigbee2mqtt/<FRIENDLY_NAME'
-    condition:
-      condition: template
-      value_template: "{{ 'single' == trigger.payload_json.click }}"
-    action:
-      entity_id: light.bedroom
-      service: light.toggle
-`
-text += '```\n'
-text += '**When changing a `friendly_name` for a device you first have to start zigbee2mqtt and after that restart Home Assistant in order to discover the new device ID.**\n\n';
-text += 'In case you **dont** want to use Home Assistant MQTT discovery you can use the configuration below.\n\n'
-
-const homeassistantConfig = (device) => {
-    const payload = {
-        platform: 'mqtt',
-        state_topic: "zigbee2mqtt/<FRIENDLY_NAME>",
-        availability_topic: "zigbee2mqtt/bridge/state",
-        ...device.discovery_payload,
-    };
-
-    if (payload.command_topic) {
-        if (payload.command_topic_prefix) {
-            payload.command_topic = `zigbee2mqtt/<FRIENDLY_NAME>/${payload.command_topic_prefix}/set`;
-        } else {
-            payload.command_topic = `zigbee2mqtt/<FRIENDLY_NAME>/set`;
-        }
-    }
-
-    delete payload.command_topic_prefix;
-
-    let yml = YAML.stringify([payload]);
-    yml = yml.replace(/(-) \n    /g, '- ');
-    yml = yml.replace('---', `${device.type}:`)
-    return yml;
-}
-
-deviceMapping.forEach((device) => {
-    text += `### ${device.model}\n`;
-    text += '```yaml\n'
-
-    const configurations = homeassistant.mapping[device.model]
-    configurations.forEach((d, i) => {
-        text += homeassistantConfig(d);
-        if (configurations.length > 1 && i < configurations.length - 1) {
-            text += '\n';
-        }
-    })
-
-    text += '```\n\n';
-});
-
-fs.writeFileSync(outputdir + '/' + file, text);

support/verify-homeassistant-mapping.js

@@ -1,13 +0,0 @@
-// Verify that there are homeassistant mappings for every device.
-const devices = require('zigbee-shepherd-converters').devices;
-const homeassistant = require('../lib/homeassistant');
-
-let failed = false;
-devices.forEach((d) => {
-    if (!homeassistant.mapping[d.model]) {
-        console.error(`Missing homeassistant mapping for '${d.model}'`);
-        failed = true;
-    }
-});
-
-process.exit(failed ? 1 : 0);

test/assets/exampleExtension.js

@@ -0,0 +1,12 @@
+class Example {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        this.mqtt = mqtt;
+        this.mqtt.publish('example/extension', 'call from constructor')
+    }
+
+    onMQTTConnected() {
+        this.mqtt.publish('example/extension', 'test')
+    }
+}
+
+module.exports = Example;

test/assets/mock-external-converter-multiple.js

@@ -0,0 +1,33 @@
+const homeassistantSwitch = {
+    type: 'switch',
+    object_id: 'switch',
+    discovery_payload: {
+        payload_off: 'OFF',
+        payload_on: 'ON',
+        value_template: '{{ value_json.state }}',
+        command_topic: true,
+    },
+};
+
+const mockDevices = [{
+    mock: 1,
+    model: 'external_converters_device_1',
+    homeassistant: [homeassistantSwitch],
+    zigbeeModel: ['external_converter_device_1'],
+    vendor: 'external_1',
+    description: 'external_1',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+}, {
+    mock: 2,
+    model: 'external_converters_device_2',
+    zigbeeModel: ['external_converter_device_2'],
+    vendor: 'external_2',
+    description: 'external_2',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+}];
+
+module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -0,0 +1,12 @@
+const mockDevice = {
+    mock: true,
+    zigbeeModel: ['external_converter_device'],
+    vendor: 'external',
+    model: 'external_converter_device',
+    description: 'external',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+};
+
+module.exports = mockDevice;

test/availability.test.js

@@ -0,0 +1,440 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
+
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Availability', () => {
+    let controller;
+
+    function getExtension() {
+        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        jest.useFakeTimers();
+        settings.set(['advanced', 'availability_timeout'], 10);
+        controller = new Controller(jest.fn(), jest.fn());
+        mocksClear.forEach((m) => m.mockClear());
+        await controller.start();
+        await flushPromises();
+    });
+
+    afterEach(async () => {
+        await controller.stop();
+        await flushPromises();
+    })
+
+    it('Should publish availabilty on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish availabilty offline when ping fails', async () => {
+        MQTT.publish.mockClear();
+        logger.error.mockClear();
+        logger.debug.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledTimes(3);
+        expect(logger.debug).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should publish availabilty online and query state on reconnect', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_enddevice/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should fail gracefully when quering state after reconnect fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('Device timedout')});
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        logger.debug.mockClear();
+        jest.advanceTimersByTime(11 * 2000);
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(logger.error).toHaveBeenCalledWith(`Failed to read state of 'bulb_color' after reconnect`);
+    });
+
+    it('Shouldnt ping again when still pinging', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        device.ping.mockImplementationOnce(async () => {await wait(100000000)});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should mark device online when receiving message while offline', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
+        MQTT.publish.mockClear();
+        getExtension().state = {};
+        const payload = {device: zigbeeHerdsman.devices.bulb_color};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should retrieve the state when device is turned on/off within availability timeout on deviceAnnounce', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+    });
+
+    it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceJoined({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not do anything when message has no device', async () => {
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
+    });
+
+    it('Should not ping devices on blocklist by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blacklist by friendly name', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blocklist by IEEE address', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should ping forced pingable devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        getExtension().state[device.ieeeAddr] = false;
+        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
+        await controller.stop();
+        await flushPromises();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+    });
+
+    it('Should not read when device has no modelID and reconnects', async () => {
+        const device = zigbeeHerdsman.devices.nomodel;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not read when device has is unsupported', async () => {
+        const device = zigbeeHerdsman.devices.unsupported_router;
+        getExtension().state[device.ieeeAddr] = true;
+        const endpoint = device.getEndpoint(1);
+        await zigbeeHerdsman.events.deviceAnnounce({device});
+        await flushPromises();
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should stop pinging device when removed', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.ping.mockClear();
+        const endpoint = device.getEndpoint(1);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should publish availability when end device joins', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
+        const payload = {device};
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/button_double_key/availability',
+          'online',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should mark non-pingable device as non-available when offline for longer than 24 hours', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const defaultLastSeen = device.lastSeen;
+        device.lastSeen = Date.now();
+        MQTT.publish.mockClear();
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        device.lastSeen = device.lastSeen - (1000 * 60 * 60 * 25);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        // Shouldn't do anything more when device is removed
+        settings.removeDevice(device.ieeeAddr);
+        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+
+        device.lastSeen = defaultLastSeen;
+    });
+
+    it('Should republish existing state on MQTT connected', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        await controller.stop();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller = new Controller(jest.fn(), jest.fn());
+        getExtension().state[device.ieeeAddr] = false;
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+          'offline',
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device is remove', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device is renamed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "bulb_color", "to": "bulb_color_new_name"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device does not exist', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/not_existing_hahaha/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/not_existing_hahaha/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/0x000b57fffec6a5b3/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/0x000b57fffec6a5b3/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bulb_color/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+});

test/bind.test.js

@@ -0,0 +1,536 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const stringify = require('json-stable-stringify-without-jsonify');
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
+
+describe('Bind', () => {
+    let controller;
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind = jest.fn();
+            endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        zigbeeHerdsman.groups.group_1.members = [];
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        debounce.mockClear();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        MQTT.publish.mockClear();
+    });
+
+    it('Should bind to device and configure reporting', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+
+        // Setup
+        const originalDeviceOutputClusters = device.getEndpoint(1).outputClusters;
+        device.getEndpoint(1).outputClusters = [...device.getEndpoint(1).outputClusters, 768];
+        const originalTargetBinds = target.binds;
+        target.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target.getClusterAttributeValue.mockImplementationOnce((cluster, value) =>  undefined);
+        mockClear(device);
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(target.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorCapabilities' ]);
+        expect(endpoint.bind).toHaveBeenCalledTimes(4);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("lightingColorCtrl", target);
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1}]);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl", "lightingColorCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).outputClusters = originalDeviceOutputClusters;
+    });
+
+    it('Should bind only specifief clusters', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color', clusters: ["genOnOff"]}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'button'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+
+        // setup
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+        const originalRemoteBinds = device.getEndpoint(1).binds;
+        device.getEndpoint(1).binds = [];
+        const originalTargetBinds = target.binds;
+        target.binds = [
+            {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'lightingColorCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}
+        ];
+
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        delete zigbeeHerdsman.devices.bulb_color.meta.configured;
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(undefined);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+
+        // Disable reporting
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1}]);
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).binds = originalRemoteBinds;
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'Coordinator'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        // Should configure reproting for device added to group
+        target1Member.configureReporting.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb');
+        await flushPromises();
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+    });
+
+    it('Should unbind from group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: '1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementation(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: target}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when source not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when target not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should bind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'button');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
+    });
+
+    it('Legacy api: Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'bulb_color');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', 'Coordinator');
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'group_1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', '1');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Legacy api: Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+
+    it('Legacy api: Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch_double/right');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Legacy api: Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/bind/remote/ep2', 'wall_switch');
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+    });
+
+    it('Legacy api: Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/unbind/remote', target);
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
+        expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genOnOff'}});
+        expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
+    });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read.mockClear();
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(4);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
+});

test/configure.test.js

@@ -0,0 +1,234 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const stringify = require('json-stable-stringify-without-jsonify');
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Configure', () => {
+    let controller;
+
+    expectRemoteConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(2);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+
+        const endpoint2 = device.getEndpoint(2);
+        expect(endpoint2.write).toHaveBeenCalledTimes(1);
+        expect(endpoint2.write).toHaveBeenCalledWith("genBasic", {"49": {"type": 25, "value": 11}}, {"disableDefaultResponse": true, "manufacturerCode": 4107});
+        expect(device.meta.configured).toBe(1);
+    }
+
+    expectRemoteNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.bind).toHaveBeenCalledTimes(0);
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+    });
+
+    it('Should configure on startup', async () => {
+        expectRemoteConfigured();
+    });
+
+    it('Should re-configure when device rejoins', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        await flushPromises();
+        mockClear(device);
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should reconfigure reporting on reportingDisabled event', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        mockClear(device);
+        expectRemoteNotConfigured();
+        controller.eventBus.emit('reportingDisabled', {device})
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should not configure twice', async () => {
+        expectRemoteConfigured();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure on zigbee message when not configured yet', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Should allow to configure via MQTT', async () => {
+        expectRemoteConfigured();
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "not_existing_device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when configure fails', async () => {
+        zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "remote"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device has no configure', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb_enddevice", transaction: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "bulb_enddevice"},"status":"error","error": "Device 'bulb_enddevice' cannot be configured","transaction":20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should allow to reconfigure manually', async () => {
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
+        await flushPromises();
+        expectRemoteConfigured();
+    });
+
+    it('Legacy api: Shouldnt manually reconfigure when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
+    });
+
+    it('Legacy api: Should skip reconfigure when device does not require this', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x0017880104e45553');
+        await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb_enddevice', device does not require this.`)
+    });
+
+    it('Should not configure when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewing = true;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteNotConfigured();
+        device.interviewing = false;
+    });
+
+    it('Should configure when not interviewCompleted', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        device.interviewCompleted = false;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectRemoteConfigured();
+        device.interviewCompleted = true;
+    });
+
+    it('Should not configure when already configuring', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(500));
+        mockClear(device);
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should configure max 3 times when fails', async () => {
+        controller.extensions.find((e) => e.constructor.name === 'Configure').attempts = {};
+        const device = zigbeeHerdsman.devices.remote;
+        delete device.meta.configured;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+    });
+});

test/controller.test.js

@@ -0,0 +1,626 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
+
+describe('Controller', () => {
+    let controller;
+    let mockExit;
+
+    beforeEach(() => {
+        zigbeeHerdsman.returnDevices.splice(0);
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+    });
+
+    it('Start controller', async () => {
+        await controller.start();
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
+        expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
+        expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
+        expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
+        expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
+        expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Start controller when permit join fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error("failed!")});
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+    });
+
+    it('Start controller with specific MQTT settings', async () => {
+        const ca = tmp.fileSync().name;
+        fs.writeFileSync(ca, "ca");
+        const key = tmp.fileSync().name;
+        fs.writeFileSync(key, "key");
+        const cert = tmp.fileSync().name;
+        fs.writeFileSync(cert, "cert");
+
+        const configuration = {
+            base_topic: "zigbee2mqtt",
+            server: "mqtt://localhost",
+            keepalive: 30,
+            ca, cert, key,
+            password: 'pass',
+            user: 'user1',
+            client_id: 'my_client_id',
+            reject_unauthorized: false,
+            version: 5,
+        }
+        settings.set(['mqtt'], configuration)
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            keepalive: 30,
+            ca: Buffer.from([99, 97]),
+            key: Buffer.from([107, 101, 121]),
+            cert: Buffer.from([99, 101, 114, 116]),
+            password: 'pass',
+            username: 'user1',
+            clientId: 'my_client_id',
+            rejectUnauthorized: false,
+            protocolVersion: 5,
+
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should generate network_key and pan_id when set to GENERATE', async () => {
+        settings.set(['advanced', 'network_key'], 'GENERATE');
+        settings.set(['advanced', 'pan_id'], 'GENERATE');
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.panID).toStrictEqual(expect.any(Number));
+        expect(data.read().advanced.network_key.length).toStrictEqual(16);
+        expect(data.read().advanced.pan_id).toStrictEqual(expect.any(Number));
+    });
+
+    it('Start controller should publish cached states', async () => {
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Start controller should not publish cached states when disabled', async () => {
+        settings.set(['advanced', 'cache_state_send_on_startup'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        const publishedTopics = MQTT.publish.mock.calls.map(m => m[0]);
+        expect(publishedTopics).toEqual(expect.not.arrayContaining(["zigbee2mqtt/bulb", "zigbee2mqtt/remote"]));
+    });
+
+    it('Start controller should not publish cached states when cache_state is false', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Log when MQTT client is unavailable', async () => {
+        jest.useFakeTimers();
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(1);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Dont publish to mqtt when client is unavailable', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        controller.mqtt.client.reconnecting = true;
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledTimes(2);
+        expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"brightness\":50,\"color\":{\"b\":10,\"g\":50,\"r\":100},\"color_temp\":370,\"dummy\":{\"1\":\"yes\",\"2\":\"no\"},\"linkquality\":99,\"state\":\"ON\"}");
+        controller.mqtt.client.reconnecting = false;
+    });
+
+    it('Load empty state when state file does not exist', async () => {
+        data.removeState();
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
+    });
+
+    it('Should remove device not on passlist on startup', async () => {
+        settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should remove non whitelisted devices on startup', async () => {
+        settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should remove device on blocklist on startup', async () => {
+        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should remove banned devices on startup', async () => {
+        settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
+    it('Start controller fails', async () => {
+        zigbeeHerdsman.start.mockImplementationOnce(() => {throw new Error('failed')});
+        await controller.start();
+        expect(mockExit).toHaveBeenCalledTimes(1);
+    });
+
+    it('Start controller with permit join true', async () => {
+        settings.set(['permit_join'], false);
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
+    });
+
+    it('Start controller with disable_led', async () => {
+        settings.set(['serial', 'disable_led'], true);
+        await controller.start();
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setLED).toHaveBeenCalledWith(false);
+    });
+
+    it('Start controller with transmit power', async () => {
+        settings.set(['experimental', 'transmit_power'], 14);
+        await controller.start();
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledWith(14);
+    });
+
+    it('Start controller and stop', async () => {
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(0, null);
+    });
+
+    it('Start controller and stop', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await controller.stop();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
+    });
+
+    it('Start controller adapter disconnects', async () => {
+        zigbeeHerdsman.stop.mockImplementationOnce(() => {throw new Error('failed')})
+        await controller.start();
+        await zigbeeHerdsman.events.adapterDisconnected();
+        await flushPromises();
+        expect(MQTT.end).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
+    });
+
+    it('Handle mqtt message', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('dummytopic', 'dummymessage');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
+    });
+
+    it('Skip MQTT messages on topic we published to', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'zigbee2mqtt/skip-this-topic' with data 'skipped'")
+        logger.debug.mockClear();
+        await controller.mqtt.publish('skip-this-topic', '', {});
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledTimes(0);
+    });
+
+    it('On zigbee event message', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1`);
+    });
+
+    it('On zigbee event message with group ID', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Received Zigbee message from 'bulb', type 'attributeReport', cluster 'genBasic', data '{"modelId":"TRADFRI bulb E27 WS opal 980lm"}' from endpoint 1 with groupID 0`);
+    });
+
+    it('Should add entities which are missing from configuration but are in database to configuration', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
+    });
+
+    it('On zigbee event message from unkown device should create it', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.notInSettings;
+        settings.removeDevice(device.ieeeAddr);
+        expect(settings.getDevice(device.ieeeAddr)).toBeNull();
+        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(settings.getDevice(device.ieeeAddr)).toStrictEqual({"ID": "0x0017880104e45519", "friendlyName": "0x0017880104e45519", "friendly_name": "0x0017880104e45519"});
+    });
+
+    it('On zigbee deviceJoined', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['blocklist'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler accept device not on blocklist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['blocklist'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept device on passlist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['passlist'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler reject device not in passlist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['passlist'], ['123']);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(false);
+    });
+
+    it('acceptJoiningDeviceHandler should prefer passlist above blocklist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        settings.set(['passlist'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('acceptJoiningDeviceHandler accept when not on blocklist and passlist', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
+        expect(await handler(device.ieeeAddr)).toBe(true);
+    });
+
+    it('Shouldnt crash when two device join events are received', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        zigbeeHerdsman.events.deviceJoined(payload);
+        zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview started', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'started'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview failed', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'failed'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee deviceInterview successful not supported', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.unsupported;
+        const payload = {device, status: 'successful'};
+        await zigbeeHerdsman.events.deviceInterview(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('On zigbee event device announce', async () => {
+        await controller.start();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        settings.set(['devices'], {})
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('On zigbee event device leave (removed from database and NOT settings)', async () => {
+        await controller.start();
+        zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+        const device = zigbeeHerdsman.devices.bulb;
+        MQTT.publish.mockClear();
+        const payload = {ieeeAddr: device.ieeeAddr};
+        await zigbeeHerdsman.events.deviceLeave(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {dummy: {1: 'yes', 2: 'no'}, color: {r: 100, g: 50, b: 10}, state: 'ON', test: undefined, test1: null, color_temp: 370, brightness: 50});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", "yes", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", "no", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test1", '', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test", '', {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+
+    it('Publish entity state attribute_json output filtered', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state attribute_json output filtered (device_options)', async () => {
+        await controller.start();
+        settings.set(['experimental', 'output'], 'attribute_and_json');
+        settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish entity state with device information', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'include_device_information'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": true}, expect.any(Function));
+
+        // Unsupported device should have model "unknown"
+        await controller.publishEntityState('unsupported2', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state without retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], false);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": false}, expect.any(Function));
+    });
+
+    it('Should publish entity state with retain', async () => {
+        await controller.start();
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Should publish entity state with expiring retention', async () => {
+        await controller.start();
+        settings.set(['mqtt', 'version'], 5);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retention'], 37);
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+    });
+
+    it('Publish entity state no empty messages', async () => {
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to disable state persistency', async () => {
+        settings.set(['advanced', 'cache_state_persistent'], false);
+        data.removeState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        await controller.stop();
+        expect(data.stateExists()).toBeFalsy();
+    });
+
+    it('Shouldnt crash when it cannot save state', async () => {
+        data.removeState();
+        await controller.start();
+        logger.error.mockClear();
+        controller.state.file = "/";
+        await controller.state.save();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+    });
+
+    it('Publish should not cache when set', async () => {
+        settings.set(['advanced', 'cache_state'], false);
+        data.writeEmptyState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Publish should not do anything for unknown entity', async () => {
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb-unknown', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should start when state is corrupted', async () => {
+        fs.writeFileSync(path.join(data.mockDir, 'state.json'), 'corrupted');
+        await controller.start();
+        await flushPromises();
+        expect(controller.state.state).toStrictEqual({});
+    });
+
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
+    });
+
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
+});

test/data.test.js

@@ -0,0 +1,26 @@
+const logger = require('./stub/logger');
+const data = require('../lib/util/data.js');
+const path = require('path');
+const tmp = require('tmp');
+const fs = require('fs');
+
+describe('Data', () => {
+    describe('Get path', () => {
+        it('Should return correct path', () => {
+            const expected = path.normalize(path.join(__dirname, '..', 'data'));
+            const actual = data.getPath();
+            expect(actual).toBe(expected);
+        });
+
+        it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
+            const expected = tmp.dirSync().name;
+            process.env.ZIGBEE2MQTT_DATA = expected;
+            data._reload();
+            const actual = data.getPath();
+            expect(actual).toBe(expected);
+            expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
+            delete process.env.ZIGBEE2MQTT_DATA;
+            data._reload();
+        });
+    });
+});

test/externalConverters.test.js

@@ -0,0 +1,135 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const path = require('path');
+const fs = require('fs');
+
+zigbeeHerdsmanConverters.addDeviceDefinition = jest.fn();
+
+const mocksClear = [zigbeeHerdsmanConverters.addDeviceDefinition, zigbeeHerdsman.permitJoin,
+    mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+jest.mock(
+    'mock-external-converter-module', () => {
+        return {
+            mock: true
+        };
+    }, {
+        virtual: true
+    });
+
+jest.mock(
+    'mock-multiple-external-converter-module', () => {
+        return [{
+            mock: 1
+        }, {
+            mock: 2
+        }];
+    }, {
+        virtual: true
+    });
+
+describe('Loads external converters', () => {
+    let controller;
+
+    beforeEach(async () => {
+        jest.useRealTimers();
+        await flushPromises();
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        data.writeEmptyState();
+        settings._reRead();
+    });
+
+    it('Does not load external converters', async () => {
+        settings.set(['external_converters'], []);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
+    });
+
+    it('Loads external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true,
+            zigbeeModel: ['external_converter_device'],
+            vendor: 'external',
+            model: 'external_converter_device',
+            description: 'external',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+    });
+
+    it('Loads multiple external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1,
+            model: 'external_converters_device_1',
+            zigbeeModel: ['external_converter_device_1'],
+            vendor: 'external_1',
+            description: 'external_1',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2,
+            model: 'external_converters_device_2',
+            zigbeeModel: ['external_converter_device_2'],
+            vendor: 'external_2',
+            description: 'external_2',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+    });
+
+    it('Loads external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-external-converter-module']);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+});

test/externalExtension.test.js

@@ -0,0 +1,102 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const tmp = require('tmp');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
+const mkdirSyncSpy = jest.spyOn(fs, 'mkdirSync');
+const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
+
+describe('User extensions', () => {
+    let controller;
+    let mockExit;
+
+    beforeEach(() => {
+        zigbeeHerdsman.returnDevices.splice(0);
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+    });
+    afterEach(() => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.rmdirSync(extensionPath, {recursive: true});
+    })
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Load user extension from api call', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "foo.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'call from constructor', { retain: false, qos: 0 }, expect.any(Function));
+        expect(mkdirSyncSpy).toHaveBeenCalledWith(extensionPath);
+    });
+
+    it('Do not load corrupted extensions', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = "definetly not a correct javascript code";
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Removes user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        const extensionFilePath = path.join(extensionPath, 'exampleExtension.js')
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), extensionFilePath)
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "exampleExtension.js"}));
+        await flushPromises();
+        expect(unlinkSyncSpy).toHaveBeenCalledWith(extensionFilePath);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "non existing.js"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/extension/remove',
+            stringify({"data":{},"status":"error","error":"Extension non existing.js doesn't exists"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/frontend.test.js

@@ -0,0 +1,214 @@
+const data = require('./stub/data');
+require('./stub/logger');
+require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = () => new Promise(setImmediate);
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mockHTTP = {
+    implementation: {
+        listen: jest.fn(),
+        on: (event, handler) => {mockHTTP.events[event] = handler},
+        close: jest.fn().mockImplementation((cb) => cb()),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockWSocket = {
+    close: jest.fn(),
+};
+
+const mockWS = {
+    implementation: {
+        clients: [],
+        on: (event, handler) => {mockWS.events[event] = handler},
+        handleUpgrade: jest.fn().mockImplementation((request, socket, head, cb) => {
+            cb(mockWSocket)
+        }),
+        emit: jest.fn(),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockNodeStatic = {
+    implementation: jest.fn(),
+    variables: {},
+    events: {},
+};
+
+jest.mock('http', () => ({
+    createServer: jest.fn().mockImplementation((onRequest) => {
+        mockHTTP.variables.onRequest = onRequest;
+        return mockHTTP.implementation;
+    }),
+}));
+
+jest.mock("serve-static", () =>
+    jest.fn().mockImplementation((path) => {
+        mockNodeStatic.variables.path = path
+        return mockNodeStatic.implementation
+    })
+);
+
+jest.mock('zigbee2mqtt-frontend', () => ({
+    getPath: () => 'my/dummy/path',
+}));
+
+jest.mock('ws', () => ({
+    OPEN: 'open',
+    Server: jest.fn().mockImplementation(() => {
+        return mockWS.implementation;
+    }),
+}));
+
+describe('Frontend', () => {
+    let controller;
+
+    beforeEach(async () => {
+        mockWS.implementation.clients = [];
+        data.writeDefaultConfiguration();
+        data.writeDefaultState();
+        settings._reRead();
+        settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
+        settings.set(['homeassistant'], true);
+        zigbeeHerdsman.devices.bulb.linkquality = 10;
+    });
+
+    afterEach(async() => {
+        delete zigbeeHerdsman.devices.bulb.linkquality;
+    });
+
+    it('Start/stop', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+
+        const mockWSClient = {
+            implementation: {
+                close: jest.fn(),
+                send: jest.fn(),
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await controller.stop();
+        expect(mockWSClient.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+    });
+
+    it('Websocket interaction', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        // Connect
+        const mockWSClient = {
+            implementation: {
+                on: (event, handler) => {mockWSClient.events[event] = handler},
+                send: jest.fn(),
+                readyState: 'open',
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, update:{state: "idle"}, update_available: false}}));
+
+        // Message
+        MQTT.publish.mockClear();
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        mockWSClient.events.message(undefined);
+        mockWSClient.events.message("");
+        mockWSClient.events.message(null);
+        await flushPromises();
+
+        // Received message on socket
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(4);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON'}}));
+
+        // Shouldnt set when not ready
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'close';
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}))
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
+
+        // Send last seen on connect
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'open';
+        settings.set(['advanced'], {last_seen: 'ISO_8601'});
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z", update:{state: "idle"}, update_available: false}}));
+    });
+
+    it('onReques/onUpgrade', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: 'http://localhost:8080/api'}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "http://localhost:8080/api"}, mockSocket, 3, expect.any(Function));
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](99);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', 99, {"url": "http://localhost:8080/api"});
+
+        mockHTTP.variables.onRequest(1, 2);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledTimes(1);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledWith(1, 2, expect.any(Function));
+    });
+
+    it('Static server', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+    });
+
+    it('Authentification', async () => {
+        const authToken = 'sample-secure-token'
+        settings.set(['frontend'], {auth_token: authToken});
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: '/api'}, mockSocket, mockWSocket);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "/api"}, mockSocket, mockWSocket, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledWith(4401, "Unauthorized");
+
+        mockWSocket.close.mockClear();
+        mockWS.implementation.emit.mockClear();
+
+        const url = `/api?token=${authToken}`;
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: url}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({url}, mockSocket, 3, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledTimes(0);
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](mockWSocket);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', mockWSocket, {url});
+
+    });
+});

test/group.test.js

@@ -0,0 +1,796 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify-without-jsonify');
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+
+const MQTT = require('./stub/mqtt');
+const Controller = require('../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const settings = require('../lib/util/settings');
+
+describe('Groups', () => {
+    let controller;
+
+    beforeEach(() => {
+        data.writeEmptyState();
+        controller = new Controller(jest.fn(), jest.fn());
+        Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        MQTT.publish.mockClear();
+        zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+    })
+
+    it('Apply group updates add', async () => {
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
+        zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
+    });
+
+    it('Apply group updates remove', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+    });
+
+    it('Apply group updates remove handle fail', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        endpoint.removeFromGroup.mockImplementationOnce(() => {throw new Error("failed!")});
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        logger.error.mockClear();
+        await controller.start();
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
+    });
+
+    it('Move to non existing group', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+    });
+
+    it('Add non standard endpoint to group with name', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
+    });
+
+    it('Add non standard endpoint to group with number', async () => {
+        const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
+    });
+
+    it('Shouldnt crash on non-existing devices', async () => {
+        logger.error.mockClear();
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
+        expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
+    });
+
+    it('Should resolve device friendly names', async () => {
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
+    });
+
+    it('Legacy api: Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Legacy api: Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Legacy api: Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+    });
+
+    it('Legacy api: Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+    });
+
+    it('Legacy api: Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+    });
+
+    it('Legacy api: Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Legacy api: Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Legacy api: Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Legacy api: Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Remove from group all deprecated', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+    });
+
+    it('Legacy api: Log when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
+    });
+
+    it('Legacy api: Log when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith("Device 'bulb_color_not_existing' does not exist");
+    });
+
+    it('Should publish group state change when a device in it changes state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not republish identical optimistic group states', async () => {
+        const device1 = zigbeeHerdsman.devices.bulb_2;
+        const device2 = zigbeeHerdsman.devices.bulb_color_2;
+        const group = zigbeeHerdsman.groups.group_tradfri_remote;
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(4);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change for group when members state change', async () => {
+        // Created for https://github.com/Koenkk/zigbee2mqtt/issues/5725
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of all members when a group changes its state, filtered', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","brightness":100}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt publish group state change when a group is not optimistic', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change of another group with shared device when a group changes its state', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
+            '3': {friendly_name: 'group_3', retain: false, devices: []}
+        });
+        await controller.start();
+        await flushPromises();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via device', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
+            '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off if all lights within turn off', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change off even when missing current state', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await controller.start();
+        await flushPromises();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+        controller.state.state = {};
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove_all',
+            stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing group', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing device', async () => {
+        await controller.start();
+        await flushPromises();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/legacy/bridgeLegacy.test.js

@@ -0,0 +1,478 @@
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+const MQTT = require('../stub/mqtt');
+const stringify = require('json-stable-stringify-without-jsonify');
+const path = require('path');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+
+
+describe('Bridge legacy', () => {
+    let controller;
+
+    beforeAll(async () => {
+        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+    })
+
+    beforeEach(() => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeDefaultState();
+        logger.info.mockClear();
+        logger.warn.mockClear();
+    });
+
+    it('Should publish bridge configuration on startup', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/config',
+          stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+    });
+
+    it('Should allow to set elapsed', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'true');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'false');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/elapsed', 'wrong');
+        await flushPromises();
+        expect(settings.get().advanced.elapsed).toBe(false);
+    });
+
+    it('Should allow whitelist', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        const bulb = zigbeeHerdsman.devices.bulb;
+        expect(settings.get().whitelist).toStrictEqual([]);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          { retain: false, qos: 0 },
+          expect.any(Function)
+        );
+
+        MQTT.publish.mockClear()
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        MQTT.events.message('zigbee2mqtt/bridge/config/whitelist', 'bulb');
+        await flushPromises();
+        expect(settings.get().whitelist).toStrictEqual([bulb_color.ieeeAddr, bulb.ieeeAddr]);
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow changing device options', async () => {
+        const bulb_color = zigbeeHerdsman.devices.bulb_color;
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', "{friendly_name: 'bulb_color'malformed: {retain: true}}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
+        );
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(
+            {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
+        );
+    });
+
+    it('Should allow permit join', async () => {
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'true');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
+        zigbeeHerdsman.permitJoin.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'false');
+        await flushPromises();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
+    });
+
+    it('Should allow to reset', async () => {
+        zigbeeHerdsman.reset.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledWith('soft');
+        zigbeeHerdsman.reset.mockImplementationOnce(() => {throw new Error('')});
+        MQTT.events.message('zigbee2mqtt/bridge/config/reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.reset).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.reset.mock.calls[1][0]).toBe('soft');
+    });
+
+    it('Should allow to set last_seen', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'ISO_8601');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('ISO_8601');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'disable');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+        MQTT.events.message('zigbee2mqtt/bridge/config/last_seen', 'notvalid');
+        await flushPromises();
+        expect(settings.get().advanced.last_seen).toBe('disable');
+    });
+
+    it('Should allow to set log_level', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'debug');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('debug');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'error');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+        MQTT.events.message('zigbee2mqtt/bridge/config/log_level', 'notvalid');
+        await flushPromises();
+        expect(logger.getLevel()).toBe('error');
+    });
+
+    it('Should allow to get devices', async () => {
+        const now = Date.now;
+        Date.now = () => 100;
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/devices/get', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/config/devices');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
+        expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "softwareBuildID": null, "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        Date.now = now;
+    });
+
+    it('Should allow to get groups', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/groups', '');
+        await flushPromises();
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
+        const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","optimistic":true,"retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","optimistic":true,"retain":false},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","optimistic":true,"retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","optimistic":true,"retain":false},{"ID":14,"devices":["power_plug"],"friendly_name":"switch_group","optimistic":true,"retain":false},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","optimistic":true,"retain":false}],"type":"groups"});
+    });
+
+    it('Should allow rename devices', async () => {
+        const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
+        MQTT.publish.mockClear();
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color')).toStrictEqual(null);
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', "{old: 'bulb_color2'newmalformed: 'bulb_color3'}");
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        await flushPromises();
+        expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
+    });
+
+    it('Should allow rename groups', async () => {
+        MQTT.publish.mockClear();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
+        await flushPromises();
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to rename last joined device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceJoined(payload);
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt rename when no device has been joined', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename_last', 'bulb_new_name');
+        await flushPromises();
+        expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
+    });
+
+    it('Should allow to add groups', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', 'new_group');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+    });
+
+    it('Should allow to add groups with json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_added', message: 'new_group'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to add groups with json specifying only id', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
+        await flushPromises();
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
+    });
+
+    it('Should allow to remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Shouldnt allow add groups without id or friendly_name in json', async () => {
+        zigbeeHerdsman.createGroup.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{}');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith('Failed to add group, missing friendly_name!');
+    });
+
+    it('Shouldnt do anything on unsupported topic', async () => {
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/not_supported', 'to_be_removed');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'device_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to force remove device', async () => {
+        controller.state.state = {'0x000b57fffec6a5b3': {brightness: 100}};
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromDatabase.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromDatabase).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(controller.state.state).toStrictEqual({});
+        expect(settings.get().ban.length).toBe(0);
+    });
+
+    it('Should allow to ban device', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        expect(settings.get().ban.length).toBe(0);
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(controller.state[device.ieeeAddr]).toBeUndefined();
+        expect(settings.getDevice('bulb_color')).toBeNull();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'device_banned', message: 'bulb_color'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+        expect(settings.get().ban).toStrictEqual(['0x000b57fffec6a5b3']);
+    });
+
+    it('Shouldnt crash when removing non-existing device', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'not_existing_123');
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith(`Cannot remove, device 'not_existing_123' does not exist`);
+    });
+
+    it('Should handle when remove fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/remove', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should handle when ban fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.removeFromNetwork.mockClear();
+        device.removeFromNetwork.mockImplementationOnce(() => {throw new Error('')})
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/config/ban', 'bulb_color');
+        await flushPromises();
+        expect(device.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false})
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to touchlink factory reset (OK)', async () => {
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
+        expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
+    });
+
+    it('Should allow to touchlink factory reset (FAILS)', async () => {
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
+
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
+        await flushPromises();
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
+        expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
+    });
+});

test/legacy/report.test.js

@@ -0,0 +1,284 @@
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
+zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45524');
+zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
+const MQTT = require('../stub/mqtt');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
+const flushPromises = () => new Promise(setImmediate);
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+
+describe('Report', () => {
+    let controller;
+
+    function expectOnOffBrightnessColorReport(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(1);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(0);
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 300, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
+    function expectOnOffBrightnessColorReportDisabled(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
+    mockClear = (device) => {
+        for (const endpoint of device.endpoints) {
+            endpoint.read.mockClear();
+            endpoint.write.mockClear();
+            endpoint.configureReporting.mockClear();
+            endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
+        }
+    }
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings._reRead();
+        data.writeEmptyState();
+        settings.set(['advanced', 'report'], true);
+        for (const device of Object.values(zigbeeHerdsman.devices)) {
+            mockClear(device);
+            delete device.meta.reporting;
+        }
+
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        mocksClear.forEach((m) => m.mockClear());
+        await flushPromises();
+    });
+
+    it('Should configure reporting on startup', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        expectOnOffBrightnessColorReport(endpoint, true);
+    });
+
+    it('Should not configure reporting on startup when disabled', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        delete device.meta.report;
+        settings.set(['advanced', 'report'], false);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should disable reporting on startup when enabled earlier', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.meta.reporting = 1;
+        const endpoint = device.getEndpoint(1);
+        settings.set(['advanced', 'report'], false);
+        mockClear(device);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expectOnOffBrightnessColorReportDisabled(endpoint, true);
+    });
+
+    it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        device.save.mockClear();
+        mockClear(device);
+        delete device.meta.reporting;
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+        expect(device.save).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not configure reporting when still configuring', async () => {
+        jest.useFakeTimers();
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => await wait(1000));
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        jest.runAllTimers();
+        jest.useRealTimers();
+    });
+
+    it('Should not mark as configured when reporting setup fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
+    it('Should not configure reporting when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        device.interviewing = true;
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
+    it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for end devices', async () => {
+        const device = zigbeeHerdsman.devices.E11_G13;
+        const endpoint = device.getEndpoint(1);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should configure reporting when deviceAnnounce message from IKEA device', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expectOnOffBrightnessColorReport(endpoint, false);
+    });
+
+    it('Should not configure reporting on device leave', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting for the CC2530 router', async () => {
+        const device = zigbeeHerdsman.devices.CC2530_ROUTER;
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        mockClear(device);
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: device.ieeeAddr});
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not configure reporting again when it already failed once', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.getEndpoint(1);
+        endpoint.bind.mockImplementationOnce(async () => {throw new Error('failed')});
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
+        const device = zigbeeHerdsman.devices.ZNLDP12LM;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        delete device.meta.reporting;
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+    });
+
+    it('Should not setup colorTemperature reporting when bulb does not support it and should read colorCapabilities when its not there yet ', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        const configuredReportings = endpoint.configuredReportings;
+        endpoint.configuredReportings = [];
+        delete device.meta.reporting;
+        mockClear(device);
+        endpoint.getClusterAttributeValue = jest.fn();
+
+        let count = 0;
+        endpoint.getClusterAttributeValue.mockImplementation((d) => {
+            count++;
+            if (count === 1) return undefined;
+            return 17;
+        });
+
+        const payload = {device};
+        await zigbeeHerdsman.events.deviceAnnounce(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.bind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        endpoint.configuredReportings = configuredReportings;
+    });
+});