1.21.1

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,8 +1,9 @@
 .eslintignore
-.eslintrc.json
+.eslintrc.js
 .git
 .github
 .gitignore
+.idea
 .npmrc
 .travis*
 .travis.yml
@@ -12,4 +13,5 @@ images
 node_modules
 scripts
 test
-update.sh
+coverage
+update.sh

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-ignore': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/FUNDING.yml

@@ -1,2 +1,4 @@
 github: [koenkk]
-custom: https://www.paypal.me/koenkk
+custom:
+  - https://www.paypal.me/koenkk
+  - https://www.buymeacoffee.com/koenkk

.github/ISSUE_TEMPLATE.md

@@ -3,7 +3,7 @@ Before submitting an issue make sure you've searched for a similar issue and rea
 
 Rules (don't ignore these, your issue will be closed without further notice):
 - English only
-- Make sure you are on the latest Zigbee2mqtt version
+- Make sure you are on the latest Zigbee2MQTT version
 - Provide a clear description of the problem
 - DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
 - Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
@@ -11,7 +11,7 @@ Rules (don't ignore these, your issue will be closed without further notice):
 Did you read the FAQ?
 - https://www.zigbee2mqtt.io/information/FAQ.html
 
-Zigbee2mqtt fails to start?
+Zigbee2MQTT fails to start?
 - https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
 
 Having issues when using a CC2531?
@@ -41,6 +41,6 @@ Bug report?
 ## How to reproduce it (minimal and precise)
 
 ## Debug Info
-Zigbee2mqtt version:
+Zigbee2MQTT version:
 Adapter hardware: CC2531, CC2530, CC26X2R1 or CC1352P-2
 Adapter firmware version:

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,17 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Questions/discussion
+    url: https://github.com/Koenkk/zigbee2mqtt/discussions/new
+    about: Ask questions, discuss about devices or show things you made
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
+  - name: Home Assistant addon issues
+    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
+    about: Issues related to the Home Assistant addon.
+  - name: FAQ
+    url: https://www.zigbee2mqtt.io/information/FAQ.html
+    about: Frequently asked questions.
+  - name: Support Chat
+    url: https://discord.gg/NyseBeK
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature request
+assignees: ''
+
+---
+
+## Is your feature request related to a problem? Please describe
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+## Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+## Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+## Additional context
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -0,0 +1,21 @@
+---
+name: New device support request
+about: Request support for a new device
+title: ''
+labels: new device support
+assignees: ''
+
+---
+
+<!--
+VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
+Clearly indicate what you tried and where you got stuck.
+-->
+
+## Information about the device + link
+..
+
+## data/database.db entry of the device
+..

.github/ISSUE_TEMPLATE/problem_report.md

@@ -0,0 +1,54 @@
+---
+name: Problem report
+about: Create a report to help us improve
+title: ''
+labels: problem
+assignees: ''
+
+---
+
+<!--
+Before submitting an issue make sure you've searched for a similar issue and read the documentation: https://www.zigbee2mqtt.io/.
+
+Rules (don't ignore these, your issue will be closed without further notice):
+- English only
+- Make sure you are on the latest Zigbee2MQTT version
+- Provide a clear description of the problem
+- DON'T copy logs directly here, post a link to https://hastebin.com/ or https://pastebin.com/.
+- Make sure you are running the latest firmware https://github.com/koenkk/z-stack-firmware.
+
+Did you read the FAQ?
+- https://www.zigbee2mqtt.io/information/FAQ.html
+
+Zigbee2MQTT fails to start?
+- https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start
+
+Having issues when using a CC2531?
+- Make sure the CC2531 is connected through a USB extension cable
+- Try the source routing firmware: https://github.com/Koenkk/Z-Stack-firmware/tree/master/coordinator/Z-Stack_Home_1.2/bin/source_routing
+- With larger networks (30/40+) devices, your CC2531 might not be powerful enough. This will cause weird issues, in this case it's advised to use a more powerful adapter: https://www.zigbee2mqtt.io/information/supported_adapters.html#texas-instruments-cc26x2r1
+
+Unsupported device?
+- https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Device does not pair or interview fails?
+- https://www.zigbee2mqtt.io/information/FAQ.html#why-does-my-device-not-or-fail-to-pair
+
+Bug report?
+- If applicable, provide steps how to reproduce the problem.
+- Provide the herdsman debug logging: https://www.zigbee2mqtt.io/information/debug.html#zigbee-herdsman-debug-logging
+
+-->
+
+<!--Start your bug report below this line-->
+
+## What happened
+
+## What did you expect to happen
+
+## How to reproduce it (minimal and precise)
+
+## Debug info
+Zigbee2MQTT version:
+Adapter hardware: CC2531, CC2530, CC26X2R1, CC1352P-2, Conbee II
+Adapter firmware version:

.github/stale.yml

@@ -1,13 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 60
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-# Label to use when marking an issue as stale
-staleLabel: stale
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+        registry-url: https://registry.npmjs.org/
+    - name: Get NodeJS Version
+      id: nodeversion
+      run: |
+        echo "::set-output name=version::$(node --version)"
+    - name: Restore node_modules cache
+      id: cache-node-modules
+      uses: actions/cache@v2
+      with:
+        path: node_modules
+        key: ${{ runner.os }}-${{ steps.nodeversion.outputs.version }}-${{ hashFiles('npm-shrinkwrap.json') }}
+    - name: Install dependencies
+      if: steps.cache-node-modules.outputs.cache-hit != 'true'
+      run: npm ci
+    - name: Build
+      run: npm run build
+    - name: Test
+      run: npm run test-with-coverage
+    - name: Lint
+      run: npm run eslint
+    - name: Docker login
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
+    - name: Docker login ghcr.io
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
+    - name: Docker setup - QEMU
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
+    - name: Docker build dev
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
+          .
+    - name: Docker build release
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: |
+        TAG="$(git describe --tags)"
+        docker buildx build \
+          --build-arg COMMIT=$(git rev-parse --short HEAD) \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
+          -f docker/Dockerfile \
+          --push \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
+          .
+    - name: Publish to npm
+      if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
+      run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
+      if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
+      run: |
+        curl -s \
+          --request POST \
+          --header "Content-Type: application/json" \
+          --data '{"definition": {"id": 9}}' \
+          -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -0,0 +1,19 @@
+name: "Close stale issues/pull requests"
+on:
+  schedule:
+  - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        stale-pr-message: 'This pull request is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 7 days'
+        days-before-stale: 30
+        days-before-close: 7
+        operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -0,0 +1,25 @@
+name: Update dependencies
+on:
+  workflow_dispatch:
+  schedule:
+  - cron: "0 0 * * 0"
+
+jobs:
+  update_deps:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update dependencies
+        branch: "update_deps"
+        title: Update dependencies

.github/workflows/update_frontend.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-frontend
+on:
+  repository_dispatch:
+    types: update_frontend
+
+jobs:
+  update_frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
+        branch: "update_frontend"
+        title: Update zigbee-frontend to ${{ github.event.client_payload.version }}

.github/workflows/update_zh.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman
+on:
+  repository_dispatch:
+    types: update_zh
+
+jobs:
+  update_zh:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
+        branch: "update_zh"
+        title: Update zigbee-herdsman to ${{ github.event.client_payload.version }}

.github/workflows/update_zhc.yml

@@ -0,0 +1,22 @@
+name: Update zigbee-herdsman-converters
+on:
+  repository_dispatch:
+    types: update_zhc
+
+jobs:
+  update_zhc:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: dev
+        ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+    - uses: actions/setup-node@v2-beta
+      with:
+        node-version: 12
+    - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
+    - uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
+        branch: "update_zhc"
+        title: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
@@ -70,4 +73,7 @@ data/database.db.backup
 data/extension
 
 # MacOS indexing file
-.DS_Store
+.DS_Store
+
+# Ignore config
+data/*.yaml

.npmignore

@@ -7,7 +7,7 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
-azure-pipelines.yaml
+babel.config.js
 
 #linters
 .jscsrc
@@ -15,11 +15,13 @@ azure-pipelines.yaml
 .eslintrc*
 .dockerignore
 .eslintignore
+.gitignore
 
 #editor settings
 .vscode
 
 #src
+lib
 docs
 docker
 images

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "files.associations": {
-        "*.yaml": "home-assistant"
-    }
-}

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing to Zigbee2MQTT
+
+Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.
+
+- Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).

README.md

@@ -5,8 +5,8 @@
     <br>
     <br>
     <div style="display: flex;">
-        <a href="https://dev.azure.com/koenkk/zigbee/_build?definitionId=5">
-            <img src="https://dev.azure.com/koenkk/zigbee/_apis/build/status/Koenkk.zigbee2mqtt?branchName=dev">
+        <a href="https://github.com/Koenkk/zigbee2mqtt/actions?query=workflow%3ACI">
+            <img src="https://github.com/koenkk/zigbee2mqtt/workflows/CI/badge.svg">
         </a>
         <a href="https://github.com/Koenkk/zigbee2mqtt/releases">
             <img src="https://img.shields.io/github/release/koenkk/zigbee2mqtt.svg">
@@ -33,7 +33,7 @@
             <img src="https://img.shields.io/npm/v/zigbee2mqtt">
         </a>
     </div>
-    <h1>Zigbee2mqtt  🌉 🐝</h1>
+    <h1>Zigbee2MQTT  🌉 🐝</h1>
     <p>
         Allows you to use your Zigbee devices <b>without</b> the vendors bridge or gateway.
     </p>
@@ -48,12 +48,12 @@ The [documentation](https://www.zigbee2mqtt.io/) provides you all the informatio
 If you aren't familiar with **Zigbee** terminology make sure you [read this](https://www.zigbee2mqtt.io/information/zigbee_network.html) to help you out.
 
 ## [Integrations](https://www.zigbee2mqtt.io/#integration)
-Zigbee2mqtt integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
+Zigbee2MQTT integrates well with (almost) every home automation solution because it uses MQTT. However the following integrations are worth mentioning:
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Hassio](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
@@ -67,7 +67,10 @@ Zigbee2mqtt integrates well with (almost) every home automation solution because
 ![Architecture](images/architecture.png)
 
 ### Internal Architecture
-Zigbee2mqtt is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the zigbee2mqtt module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2mqtt also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
 
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.

azure-pipelines.yaml

@@ -1,88 +0,0 @@
-trigger:
-  tags:
-    include:
-    - '*'
-  branches:
-    include:
-    - '*'
-
-pool:
-  vmImage: 'ubuntu-latest'
-
-steps:
-
-# Prepare
-- task: NodeTool@0
-  inputs:
-    versionSpec: '12.x'
-  displayName: 'Install Node.js'
-- bash: npm ci
-  displayName: 'Install dependencies'
-
-# Verify
-- bash: npm run test-with-coverage
-  displayName: 'Test'
-- bash: npm run eslint
-  displayName: 'Lint'
-
-# NPM
-- task: Npm@1
-  inputs:
-    command: publish
-    publishEndpoint: npm
-  displayName: 'Publish to npm'
-  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
-
-# Docker
-- task: DockerInstaller@0
-  displayName: Docker install
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-  inputs:
-    dockerVersion: 19.03.5
-    releaseType: stable
-- task: Docker@2
-  displayName: Docker registry login
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-  inputs:
-    command: login
-    containerRegistry: docker_registry
-- bash: |
-    sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
-    sudo chmod a+x /usr/local/bin/buildx
-    docker run --rm --privileged hypriot/qemu-register:v2.7.0
-    buildx create --use
-    buildx ls
-  displayName: 'Docker setup'
-  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
-- bash: |
-    buildx build \
-      --build-arg COMMIT=$(git rev-parse --short HEAD) \
-      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
-      -f docker/Dockerfile \
-      --push \
-      -t koenkk/zigbee2mqtt:latest-dev \
-      .
-  displayName: 'Docker build dev'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))
-- bash: |
-    TAG="$(git describe --tags)"
-    buildx build \
-      --build-arg COMMIT=$(git rev-parse --short HEAD) \
-      --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
-      -f docker/Dockerfile \
-      --push \
-      -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
-      .
-  displayName: 'Docker build release'
-  condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'), ne(variables['Build.Reason'], 'PullRequest'))
-
-# Trigger Home Assistant Zigbee2mqtt addon build
-- bash: |
-    curl -s \
-        --request POST \
-        --header "Content-Type: application/json" \
-        --data '{"definition": {"id": 1}}' \
-        -u "$(HASSIO_TRIGGER_SECRET)" \
-        "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
-  displayName: 'Trigger Hass.io build'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), ne(variables['Build.Reason'], 'PullRequest'))

babel.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+};

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -1,30 +1,37 @@
-FROM node:12-alpine as base
+FROM node:14-alpine3.12 as base
 
 WORKDIR /app
-RUN apk add --no-cache tzdata eudev
+RUN apk add --no-cache tzdata eudev tini
 
-COPY package.json .
+COPY package.json ./
 
-# Dependencies
-FROM base as dependencies
+# Dependencies and build
+FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json .
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
 
-RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
-    npm ci --production && \
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
-COPY --from=dependencies /app/node_modules ./node_modules
-COPY lib ./lib
-COPY index.js docker/run.sh data/configuration.yaml ./
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
+COPY LICENSE index.js data/configuration.yaml ./
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 
-RUN chmod +x /app/run.sh
 RUN mkdir /app/data
 
 ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+RUN echo "$COMMIT" > dist/.hash
 
-ENTRYPOINT ["./run.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

docker/docker-entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+set -e
 
 if [ ! -z "$ZIGBEE2MQTT_DATA" ]; then
     DATA="$ZIGBEE2MQTT_DATA"
@@ -13,4 +14,4 @@ if [ ! -f "$DATA/configuration.yaml" ]; then
     cp /app/configuration.yaml "$DATA/configuration.yaml"
 fi
 
-exec npm start
+exec "$@"

index.js

@@ -1,23 +1,115 @@
 const semver = require('semver');
 const engines = require('./package.json').engines;
+const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
+require('source-map-support').install();
 
-const version = engines.node;
-if (!semver.satisfies(process.version, version)) {
-    console.log(`\t\tZigbee2mqtt requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
-}
-
-const Controller = require('./lib/controller');
-const controller = new Controller();
-controller.start();
-
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
+let controller;
 let stopping = false;
 
-function handleQuit() {
-    if (!stopping) {
-        stopping = true;
-        controller.stop();
+const hashFile = path.join(__dirname, 'dist', '.hash');
+
+async function restart() {
+    await stop(indexJsRestart);
+    await start();
+}
+
+async function exit(code, reason) {
+    if (reason !== indexJsRestart) {
+        process.exit(code);
     }
 }
+
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
+async function start() {
+    await checkDist();
+
+    const version = engines.node;
+    if (!semver.satisfies(process.version, version)) {
+        console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    }
+
+    // Validate settings
+    const settings = require('./dist/util/settings');
+    settings.reRead();
+    const errors = settings.validate();
+    if (errors.length > 0) {
+        console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+        console.log('            READ THIS CAREFULLY\n');
+        console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+        for (const error of errors) {
+            console.log(`- ${error}`);
+        }
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+        exit(1);
+    }
+
+    const Controller = require('./dist/controller');
+    controller = new Controller(restart, exit);
+    await controller.start();
+}
+
+async function stop(reason=null) {
+    await controller.stop(reason);
+}
+
+async function handleQuit() {
+    if (!stopping && controller) {
+        stopping = true;
+        await stop();
+    }
+}
+
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -4,13 +4,12 @@ const EventBus = require('./eventBus');
 const State = require('./state');
 const logger = require('./util/logger');
 const settings = require('./util/settings');
-const objectAssignDeep = require('object-assign-deep');
 const utils = require('./util/utils');
-const fs = require('fs');
-const data = require('./util/data');
-const path = require('path');
+const stringify = require('json-stable-stringify-without-jsonify');
+const assert = require('assert');
 
 // Extensions
+const ExtensionFrontend = require('./extension/frontend');
 const ExtensionPublish = require('./extension/publish');
 const ExtensionReceive = require('./extension/receive');
 const ExtensionNetworkMap = require('./extension/networkMap');
@@ -22,24 +21,40 @@ const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
 const ExtensionBridge = require('./extension/bridge');
 const ExtensionGroups = require('./extension/groups');
 const ExtensionAvailability = require('./extension/availability');
+const ExtensionAvailabilityNew = require('./extension/availabilityNew');
 const ExtensionBind = require('./extension/bind');
-const ExtensionReport = require('./extension/report');
+const ExtensionReport = require('./extension/legacy/report');
 const ExtensionOnEvent = require('./extension/onEvent');
 const ExtensionOTAUpdate = require('./extension/otaUpdate');
+const ExtensionExternalConverters = require('./extension/externalConverters');
+const ExtensionExternalExtension = require('./extension/externalExtension');
+
+const AllExtensions = [
+    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
+    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
+    ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension, ExtensionAvailabilityNew,
+];
 
 class Controller {
-    constructor() {
+    constructor(restartCallback, exitCallback) {
         this.zigbee = new Zigbee();
         this.mqtt = new MQTT();
         this.eventBus = new EventBus();
         this.state = new State(this.eventBus);
+        this.restartCallback = restartCallback;
+        this.exitCallback = exitCallback;
 
         this.publishEntityState = this.publishEntityState.bind(this);
+        this.enableDisableExtension = this.enableDisableExtension.bind(this);
         this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
+        this.addExtension = this.addExtension.bind(this);
 
         // Initialize extensions.
-        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
+        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
         this.extensions = [
+            new ExtensionBridge(...args),
             new ExtensionPublish(...args),
             new ExtensionReceive(...args),
             new ExtensionDeviceGroupMembership(...args),
@@ -49,18 +64,19 @@ class Controller {
             new ExtensionBind(...args),
             new ExtensionOnEvent(...args),
             new ExtensionOTAUpdate(...args),
+            new ExtensionReport(...args),
         ];
 
-        if (settings.get().experimental.new_api) {
-            this.extensions.push(new ExtensionBridge(...args));
+        if (settings.get().frontend) {
+            this.extensions.push(new ExtensionFrontend(...args));
         }
 
         if (settings.get().advanced.legacy_api) {
             this.extensions.push(new ExtensionBridgeLegacy(...args));
         }
 
-        if (settings.get().advanced.report) {
-            this.extensions.push(new ExtensionReport(...args));
+        if (settings.get().external_converters.length) {
+            this.extensions.push(new ExtensionExternalConverters(...args));
         }
 
         if (settings.get().homeassistant) {
@@ -72,49 +88,44 @@ class Controller {
             this.extensions.push(new ExtensionSoftReset(...args));
         }
 
-        if (settings.get().advanced.availability_timeout) {
+        if (settings.get().experimental.availability_new) {
+            this.extensions.push(new ExtensionAvailabilityNew(...args));
+        } else if (settings.get().advanced.availability_timeout) {
             this.extensions.push(new ExtensionAvailability(...args));
         }
 
-        const extensionPath = data.joinPath('extension');
-        if (fs.existsSync(extensionPath)) {
-            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
-            for (const extension of extensions) {
-                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
-                this.extensions.push(new Extension(...args, settings, logger));
-            }
-        }
+        this.extensions.push(new ExtensionExternalExtension(...args));
     }
 
     async start() {
-        const settingsErrors = settings.validate();
-        if (settingsErrors) {
-            logger.error(`Refusing to start, configuration.yaml is not valid, found the following errors:`);
-            for (const error of settingsErrors) {
-                logger.error(`\t - ${error}`);
-            }
-            logger.error(
-                `If you don't know how to solve this, read https://www.zigbee2mqtt.io/configuration/configuration.html`,
-            );
-            process.exit(1);
-        }
-
         this.state.start();
+        logger.logOutput();
 
-        const info = await utils.getZigbee2mqttVersion();
-        logger.info(`Starting zigbee2mqtt version ${info.version} (commit #${info.commitHash})`);
+        const info = await utils.getZigbee2MQTTVersion();
+        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
 
         // Start zigbee
+        let startResult;
         try {
-            await this.zigbee.start();
+            startResult = await this.zigbee.start();
             this.callExtensionMethod('onZigbeeStarted', []);
             this.zigbee.on('event', this.onZigbeeEvent.bind(this));
             this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
         } catch (error) {
             logger.error('Failed to start zigbee');
+            // eslint-disable-next-line
+            logger.error('Check https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start for possible solutions');
             logger.error('Exiting...');
             logger.error(error.stack);
-            process.exit(1);
+            this.exitCallback(1);
+        }
+
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
         }
 
         // Log zigbee clients on startup
@@ -122,14 +133,10 @@ class Controller {
         logger.info(`Currently ${devices.length} devices are joined:`);
         for (const device of devices) {
             const entity = this.zigbee.resolveEntity(device);
-            logger.info(
-                (entity.settings ? entity.settings.friendlyName : entity.device.ieeeAddr) +
-                ` (${entity.device.ieeeAddr}): ` +
-                (entity.definition ?
-                    `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description} ` :
-                    'Not supported ') +
-                `(${entity.device.type})`,
-            );
+            const model = entity.definition ?
+                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
+                'Not supported';
+            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
         }
 
         // Enable zigbee join.
@@ -139,14 +146,18 @@ class Controller {
             logger.warn('Set `permit_join` to `false` once you joined all devices.');
         }
 
-        await this.zigbee.permitJoin(settings.get().permit_join);
+        try {
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
 
         // MQTT
         this.mqtt.on('message', this.onMQTTMessage.bind(this));
         await this.mqtt.connect();
 
         // Send all cached states.
-        if (settings.get().advanced.cache_state) {
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
             for (const device of this.zigbee.getClients()) {
                 if (this.state.exists(device.ieeeAddr)) {
                     this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
@@ -165,7 +176,30 @@ class Controller {
         await this.callExtensionMethod('onMQTTConnected', []);
     }
 
-    async stop() {
+    async enableDisableExtension(enable, name) {
+        if (!enable) {
+            const extension = this.extensions.find((e) => e.constructor.name === name);
+            if (extension) {
+                await this.callExtensionMethod('stop', [], [extension]);
+                this.extensions.splice(this.extensions.indexOf(extension), 1);
+            }
+        } else {
+            const Extension = AllExtensions.find((e) => e.name === name);
+            assert(Extension, `Extension '${name}' does not exist`);
+            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+                this.enableDisableExtension, this.restartCallback, this.addExtension);
+
+            this.extensions.push(extension);
+            await this.callExtensionMethod('onZigbeeStarted', [], [extension]);
+            await this.callExtensionMethod('onMQTTConnected', [], [extension]);
+        }
+    }
+
+    addExtension(extension) {
+        this.extensions.push(extension);
+    }
+
+    async stop(reason=null) {
         // Call extensions
         await this.callExtensionMethod('stop', []);
 
@@ -175,10 +209,11 @@ class Controller {
 
         try {
             await this.zigbee.stop();
-            process.exit(0);
+            logger.info('Stopped Zigbee2MQTT');
+            this.exitCallback(0, reason);
         } catch (error) {
-            logger.error('Failed to stop zigbee');
-            process.exit(1);
+            logger.error('Failed to stop Zigbee2MQTT');
+            this.exitCallback(1, reason);
         }
     }
 
@@ -189,9 +224,14 @@ class Controller {
 
     async onZigbeeEvent(type, data) {
         const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !resolvedEntity.settings && data.device.type !== 'Coordinator') {
+        if (data.device && data.device.type === 'Coordinator') {
+            logger.debug('Ignoring message from coordinator');
+            return;
+        }
+
+        if (data.device && !settings.getDevice(data.device.ieeeAddr)) {
             // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            resolvedEntity.settings = settings.addDevice(data.device.ieeeAddr);
+            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
         }
 
         const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
@@ -199,7 +239,7 @@ class Controller {
         if (type === 'message') {
             logger.debug(
                 `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
-                `, data '${JSON.stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
                 (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
             );
         } else if (type === 'deviceJoined') {
@@ -213,7 +253,8 @@ class Controller {
                     logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
                 } else {
                     logger.warn(
-                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
+                        `Device '${name}' with Zigbee model '${data.device.modelID}' and manufacturer name ` +
+                        `'${data.device.manufacturerName}' is NOT supported, ` +
                         `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
                     );
                 }
@@ -227,15 +268,27 @@ class Controller {
             }
         } else if (type === 'deviceAnnounce') {
             logger.debug(`Device '${name}' announced itself`);
+        } else if (type === 'deviceNetworkAddressChanged') {
+            logger.debug(`Device '${name}' changed network address`);
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
+                const settingsDevice = settings.getDevice(data.ieeeAddr);
+                logger.warn(`Device '${settingsDevice ? settingsDevice.friendlyName : data.ieeeAddr}' ` +
+                    `left the network`);
+                this.state.remove(data.ieeeAddr);
             }
         }
 
         // Call extensions
-        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+        const result = await this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
+
+        // In case this message is not handled by the receive extension and last_seen is enabled, publish
+        // a message to update the last_seen.
+        if (name && !result[ExtensionReceive.prototype.constructor.name] &&
+            settings.get().advanced.last_seen !== 'disable' && type !== 'deviceLeave') {
+            this.publishEntityState(name, {});
+        }
     }
 
     onMQTTMessage(payload) {
@@ -254,25 +307,21 @@ class Controller {
         }
 
         let messagePayload = {...payload};
-        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
-            this.state.get(resolvedEntity.settings.ID) : {};
-        const newState = objectAssignDeep.noMutate(currentState, payload);
 
         // Update state cache with new state.
-        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
+        const newState = this.state.set(resolvedEntity.settings.ID, payload, stateChangeReason);
 
         if (settings.get().advanced.cache_state) {
             // Add cached state to payload
             messagePayload = newState;
         }
 
-        const deviceOptions = settings.get().device_options;
         const options = {
-            retain: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retain', false),
-            qos: utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'qos', 0),
+            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
+            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
         };
 
-        const retention = utils.getObjectsProperty([resolvedEntity.settings, deviceOptions], 'retention', false);
+        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
         if (retention !== false) {
             options.properties = {messageExpiryInterval: retention};
         }
@@ -292,40 +341,46 @@ class Controller {
             attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
         }
 
+        // Add lastseen
         const lastSeen = settings.get().advanced.last_seen;
         if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
             messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
         }
 
-        // filter mqtt message attributes
-        if (deviceOptions.filtered_attributes) {
-            deviceOptions.filtered_attributes.forEach((a) => delete messagePayload[a]);
+        // Add device linkquality.
+        if (resolvedEntity.type === 'device' && resolvedEntity.device.linkquality !== undefined) {
+            messagePayload.linkquality = resolvedEntity.device.linkquality;
         }
+
+        // filter mqtt message attributes
         if (resolvedEntity.settings.filtered_attributes) {
             resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
         }
 
-        if (Object.entries(messagePayload).length) {
-            if (settings.get().experimental.output === 'attribute_and_json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
-                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-            } else if (settings.get().experimental.output === 'json') {
-                await this.mqtt.publish(resolvedEntity.name, JSON.stringify(messagePayload), options);
-            } else {
-                /* istanbul ignore else */
-                if (settings.get().experimental.output === 'attribute') {
-                    await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-                }
+        for (const extension of this.extensions) {
+            if (extension.adjustMessagePayloadBeforePublish) {
+                extension.adjustMessagePayloadBeforePublish(resolvedEntity, messagePayload);
             }
         }
 
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
+        if (Object.entries(messagePayload).length) {
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
+            }
+
+            if (output === 'attribute_and_json' || output === 'attribute') {
+                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
+            }
+        }
+
+        this.eventBus.emit('publishEntityState', {messagePayload, entity: resolvedEntity, stateChangeReason, payload});
     }
 
     async iteratePayloadAttributeOutput(topicRoot, payload, options) {
         for (const [key, value] of Object.entries(payload)) {
             let subPayload = value;
-            let message;
+            let message = null;
 
             // Special cases
             if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
@@ -333,23 +388,28 @@ class Controller {
             }
 
             // Check Array first, since it is also an Object
-            if (Array.isArray(subPayload)) {
+            if (subPayload === null || subPayload === undefined) {
+                message = '';
+            } else if (Array.isArray(subPayload)) {
                 message = subPayload.map((x) => `${x}`).join(',');
             } else if (typeof subPayload === 'object') {
-                return this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
+                this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
             } else {
-                message = typeof subPayload === 'string' ? subPayload : JSON.stringify(subPayload);
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
             }
 
-            await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            if (message !== null) {
+                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            }
         }
     }
 
-    async callExtensionMethod(method, parameters) {
-        for (const extension of this.extensions) {
+    async callExtensionMethod(method, parameters, extensions=null) {
+        const result = {};
+        for (const extension of extensions || this.extensions) {
             if (extension[method]) {
                 try {
-                    await extension[method](...parameters);
+                    result[extension.constructor.name] = await extension[method](...parameters);
                 } catch (error) {
                     /* istanbul ignore next */
                     logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
@@ -360,6 +420,7 @@ class Controller {
                 }
             }
         }
+        return result;
     }
 }
 

lib/eventBus.js

@@ -7,18 +7,35 @@ const allowedEvents = [
     'groupRenamed', // Group has been renamed
     'publishEntityState', // Entity state will be published
     'stateChange', // Entity changes its state
+    'groupMembersChanged', // Members of a group has been changed
+    'reportingDisabled', // Reporting is disabled for a device
+    'devicesChanged', // Devices changed
 ];
 
 class EventBus extends events.EventEmitter {
+    constructor() {
+        super();
+        this.callbackByExtension = {};
+    }
+
     emit(event, data) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
         super.emit(event, data);
     }
 
-    on(event, callback) {
+    on(event, callback, extension) {
         assert(allowedEvents.includes(event), `Event '${event}' not supported`);
+        assert(extension, `Extension cannot be null`);
+        if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
+        this.callbackByExtension[extension].push({event, callback});
         super.on(event, callback);
     }
+
+    removeListenersExtension(extension) {
+        for (const entry of this.callbackByExtension[extension] || []) {
+            super.removeListener(entry.event, entry.callback);
+        }
+    }
 }
 
 module.exports = EventBus;

lib/extension/availability.js

@@ -3,15 +3,24 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const Extension = require('./extension');
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/(.*)/availability`);
 
 // Pingable end devices, some end devices should be pinged
 // e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
 const pingableEndDevices = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-N1EA'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
 ];
 
 const Hours25 = 1000 * 60 * 60 * 25;
+const AvailabilityLagRatio = 0.1;
+
+
+function timeoutLag(timeout, ratio) {
+    const lag = timeout * ratio;
+    return Math.floor(Math.random() * Math.floor(lag));
+}
 
 /**
  * This extensions pings devices to check if they are online.
@@ -24,22 +33,40 @@ class Availability extends Extension {
         this.timers = {};
         this.state = {};
 
-        this.blacklist = settings.get().advanced.availability_blacklist.map((e) => settings.getEntity(e).ID);
-        this.whitelist = settings.get().advanced.availability_whitelist.map((e) => settings.getEntity(e).ID);
+        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
+        this.eventBus.on('deviceRenamed', (data) => this.onDeviceRenamed(data), this.constructor.name);
+
+        this.blocklist = settings.get().advanced.availability_blocklist
+            .concat(settings.get().advanced.availability_blacklist)
+            .map((e) => settings.getEntity(e).ID);
+
+        this.passlist = settings.get().advanced.availability_passlist
+            .concat(settings.get().advanced.availability_whitelist)
+            .map((e) => settings.getEntity(e).ID);
     }
 
-    inWhitelistOrNotInBlacklist(device) {
+    onDeviceRenamed(data) {
+        this.mqtt.publish(`${data.from}/availability`, null, {retain: true, qos: 0});
+    }
+
+    onDeviceRemoved(resolvedEntity) {
+        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
+        delete this.state[resolvedEntity.device.ieeeAddr];
+        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
+    }
+
+    inPasslistOrNotInBlocklist(device) {
         const ieeeAddr = device.ieeeAddr;
         const deviceSettings = settings.getDevice(ieeeAddr);
         const name = deviceSettings.friendlyName;
 
-        // Whitelist is not empty and device is in it, enable availability
-        if (this.whitelist.length > 0) {
-            return this.whitelist.includes(ieeeAddr) || (name && this.whitelist.includes(name));
+        // Passlist is not empty and device is in it, enable availability
+        if (this.passlist.length > 0) {
+            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
         }
 
-        // Device is on blacklist, disable availability
-        if (this.blacklist.includes(ieeeAddr) || (name && this.blacklist.includes(name))) {
+        // Device is on blocklist, disable availability
+        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
             return false;
         }
 
@@ -52,7 +79,7 @@ class Availability extends Extension {
         }
 
         // Device is a mains powered router
-        return utils.isRouter(device) && !utils.isBatteryPowered(device);
+        return device.type === 'Router' && device.powerSource !== 'Battery';
     }
 
     onMQTTConnected() {
@@ -61,18 +88,26 @@ class Availability extends Extension {
             const ieeeAddr = device.ieeeAddr;
             this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
 
-            if (this.inWhitelistOrNotInBlacklist(device)) {
+            if (this.inPasslistOrNotInBlocklist(device)) {
                 if (this.isPingable(device)) {
                     this.setTimerPingable(device);
                 } else {
                     this.timers[ieeeAddr] = setInterval(() => {
                         this.handleIntervalNotPingable(device);
-                    }, utils.secondsToMilliseconds(300));
+                    }, utils.seconds(300));
                 }
             }
         }
     }
 
+    async onMQTTMessage(topic, message) {
+        // Clear topics for non-existing devices
+        const match = topic.match(topicRegex);
+        if (match && (!this.zigbee.resolveEntity(match[1]) || this.zigbee.resolveEntity(match[1]).name !== match[1])) {
+            this.mqtt.publish(`${match[1]}/availability`, null, {retain: true, qos: 0});
+        }
+    }
+
     async handleIntervalPingable(device) {
         // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
         const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
@@ -109,16 +144,15 @@ class Availability extends Extension {
     }
 
     setTimerPingable(device) {
-        if (this.timers[device.ieeeAddr]) {
-            clearTimeout(this.timers[device.ieeeAddr]);
-        }
-
+        const timeout = this.availability_timeout + timeoutLag(this.availability_timeout, AvailabilityLagRatio);
+        clearTimeout(this.timers[device.ieeeAddr]);
         this.timers[device.ieeeAddr] = setTimeout(async () => {
             await this.handleIntervalPingable(device);
-        }, utils.secondsToMilliseconds(this.availability_timeout));
+        }, utils.seconds(timeout));
     }
 
     async stop() {
+        super.stop();
         for (const timer of Object.values(this.timers)) {
             clearTimeout(timer);
         }
@@ -129,13 +163,11 @@ class Availability extends Extension {
     async onReconnect(device) {
         const resolvedEntity = this.zigbee.resolveEntity(device);
         if (resolvedEntity && resolvedEntity.definition) {
-            const used = [];
             try {
                 for (const key of ['state', 'brightness', 'color', 'color_temp']) {
                     const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
-                    if (converter && !used.includes(converter)) {
+                    if (converter) {
                         await converter.convertGet(device.endpoints[0], key, {});
-                        used.push(converter);
                     }
                 }
             } catch (error) {
@@ -150,9 +182,8 @@ class Availability extends Extension {
             this.onReconnect(device);
         }
 
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
-        const topic = `${name}/availability`;
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        const topic = `${resolvedEntity.name}/availability`;
         const payload = available ? 'online' : 'offline';
         if (this.state[ieeeAddr] !== available || force) {
             this.state[ieeeAddr] = available;
@@ -166,7 +197,7 @@ class Availability extends Extension {
             return;
         }
 
-        if (this.inWhitelistOrNotInBlacklist(device)) {
+        if (this.inPasslistOrNotInBlocklist(device)) {
             this.publishAvailability(data.device, true);
 
             if (this.isPingable(device)) {

lib/extension/availabilityNew.ts

@@ -0,0 +1,206 @@
+import ExtensionTS from './extensionts';
+import logger from '../util/logger';
+import {sleep, isAvailabilityEnabledForDevice, hours, minutes, seconds} from '../util/utils';
+import * as settings from '../util/settings';
+import debounce from 'debounce';
+
+// TODO
+// - Enable for HA addon
+// - Add to setting schema (when old availability is removed)
+class AvailabilityNew extends ExtensionTS {
+    private timers: {[s: string]: NodeJS.Timeout} = {};
+    private availabilityCache: {[s: string]: boolean} = {};
+    private retrieveStateDebouncers: {[s: string]: () => void} = {};
+    private pingQueue: ResolvedDevice[] = [];
+    private pingQueueExecuting = false;
+
+    constructor(zigbee: TempZigbee, mqtt: TempMQTT, state: TempState,
+        publishEntityState: TempPublishEntityState, eventBus: TempEventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.lastSeenChanged = this.lastSeenChanged.bind(this);
+        logger.warn('Using experimental new availability feature');
+    }
+
+    private getTimeout(rd: ResolvedDevice): number {
+        if (typeof rd.settings.availability === 'object' && rd.settings.availability?.timeout != null) {
+            return minutes(rd.settings.availability.timeout);
+        }
+
+        const key = this.isActiveDevice(rd) ? 'active' : 'passive';
+        const availabilitySettings = settings.get().availability;
+        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
+            return minutes(availabilitySettings[key]?.timeout);
+        }
+
+        return key === 'active' ? minutes(10) : hours(25);
+    }
+
+    private isActiveDevice(rd: ResolvedDevice): boolean {
+        return (rd.device.type === 'Router' && rd.device.powerSource !== 'Battery') ||
+            rd.device.powerSource === 'Mains (single phase)';
+    }
+
+    private isAvailable(rd: ResolvedDevice): boolean {
+        const ago = Date.now() - rd.device.lastSeen;
+        return ago < this.getTimeout(rd);
+    }
+
+    private resetTimer(rd: ResolvedDevice): void {
+        clearTimeout(this.timers[rd.device.ieeeAddr]);
+
+        // If the timer triggers, the device is not avaiable anymore otherwise resetTimer already have been called
+        if (this.isActiveDevice(rd)) {
+            // If device did not check in, ping it, if that fails it will be marked as offline
+            this.timers[rd.device.ieeeAddr] = setTimeout(
+                () => this.addToPingQueue(rd), this.getTimeout(rd) + seconds(1));
+        } else {
+            this.timers[rd.device.ieeeAddr] = setTimeout(
+                () => this.publishAvailability(rd, true), this.getTimeout(rd) + seconds(1));
+        }
+    }
+
+    private addToPingQueue(rd: ResolvedDevice): void {
+        this.pingQueue.push(rd);
+        this.pingQueueExecuteNext();
+    }
+
+    private removeFromPingQueue(rd: ResolvedDevice): void {
+        const index = this.pingQueue.findIndex((r) => r.device.ieeeAddr === rd.device.ieeeAddr);
+        index != -1 && this.pingQueue.splice(index, 1);
+    }
+
+    private async pingQueueExecuteNext(): Promise<void> {
+        if (this.pingQueue.length === 0 || this.pingQueueExecuting) return;
+        this.pingQueueExecuting = true;
+
+        const rd = this.pingQueue[0];
+        let pingedSuccessfully = false;
+        const available = this.availabilityCache[rd.device.ieeeAddr] || this.isAvailable(rd);
+        const attempts = available ? 2 : 1;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                // Enable recovery if device is marked as available and first ping fails.
+                const disableRecovery = !(i == 1 && available);
+                await rd.device.ping(disableRecovery);
+                pingedSuccessfully = true;
+                logger.debug(`Succesfully pinged '${rd.name}' (attempt ${i + 1}/${attempts})`);
+                break;
+            } catch (error) {
+                logger.error(`Failed to ping '${rd.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                // Try again in 3 seconds.
+                const lastAttempt = i - 1 === attempts;
+                !lastAttempt && await sleep(3);
+            }
+        }
+
+        this.publishAvailability(rd, !pingedSuccessfully);
+        this.resetTimer(rd);
+        this.removeFromPingQueue(rd);
+
+        // Sleep 2 seconds before executing next ping
+        await sleep(2);
+        this.pingQueueExecuting = false;
+        this.pingQueueExecuteNext();
+    }
+
+    override onMQTTConnected(): void {
+        for (const device of this.zigbee.getClients()) {
+            const rd = this.zigbee.resolveEntity(device) as ResolvedDevice;
+            if (isAvailabilityEnabledForDevice(rd, settings.get())) {
+                // Publish initial availablility
+                this.publishAvailability(rd, true);
+
+                this.resetTimer(rd);
+
+                // If an active device is initially unavailable, ping it.
+                if (this.isActiveDevice(rd) && !this.isAvailable(rd)) {
+                    this.addToPingQueue(rd);
+                }
+            }
+        }
+    }
+
+    override onZigbeeStarted(): void {
+        this.zigbee.on('lastSeenChanged', this.lastSeenChanged);
+    }
+
+    override onZigbeeEvent(type: ZigbeeEventType, data: ZigbeeEventData, re: ResolvedEntity): void {
+        /* istanbul ignore else */
+        if (type === 'deviceLeave') {
+            clearTimeout(this.timers[data.ieeeAddr]);
+        } else if (type === 'deviceAnnounce') {
+            this.retrieveState(re as ResolvedDevice);
+        }
+    }
+
+    private publishAvailability(rd: ResolvedDevice, logLastSeen: boolean): void {
+        if (logLastSeen) {
+            const ago = Date.now() - rd.device.lastSeen;
+            if (this.isActiveDevice(rd)) {
+                logger.debug(
+                    `Active device '${rd.name}' was last seen '${(ago / minutes(1)).toFixed(2)}' minutes ago.`);
+            } else {
+                logger.debug(`Passive device '${rd.name}' was last seen '${(ago / hours(1)).toFixed(2)}' hours ago.`);
+            }
+        }
+
+        const available = this.isAvailable(rd);
+        if (this.availabilityCache[rd.device.ieeeAddr] == available) {
+            return;
+        }
+
+        if (rd.device.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[rd.device.ieeeAddr] === false) {
+            logger.debug(`Device '${rd.name}' reconnected`);
+            this.retrieveState(rd);
+        }
+
+        const topic = `${rd.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        this.availabilityCache[rd.device.ieeeAddr] = available;
+        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+    }
+
+    private lastSeenChanged(data: {device: Device}): void {
+        const rd = this.zigbee.resolveEntity(data.device) as ResolvedDevice;
+        if (isAvailabilityEnabledForDevice(rd, settings.get())) {
+            // Remove from ping queue, not necessary anymore since we know the device is online.
+            this.removeFromPingQueue(rd);
+            this.resetTimer(rd);
+            this.publishAvailability(rd, false);
+        }
+    }
+
+    override stop(): void {
+        Object.values(this.timers).forEach((t) => clearTimeout(t));
+        this.zigbee.removeListener('lastSeenChanged', this.lastSeenChanged);
+        super.stop();
+    }
+
+    private retrieveState(rd: ResolvedDevice): void {
+        /**
+         * Retrieve state of a device in a debounced manner, this function is called on a 'deviceAnnounce' which a
+         * device can send multiple times after each other.
+         */
+        if (rd.definition && !rd.device.interviewing && !this.retrieveStateDebouncers[rd.device.ieeeAddr]) {
+            this.retrieveStateDebouncers[rd.device.ieeeAddr] = debounce(async () => {
+                try {
+                    logger.debug(`Retrieving state of '${rd.name}' after reconnect`);
+                    // Color and color temperature converters do both, only needs to be called once.
+                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
+                    for (const keys of keySet) {
+                        const converter = rd.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
+                        await converter?.convertGet?.(rd.endpoint, keys[0],
+                            {message: this.state.get(rd.device.ieeeAddr) || {}, mapped: rd.definition});
+                    }
+                } catch (error) {
+                    logger.error(`Failed to read state of '${rd.name}' after reconnect (${error.message})`);
+                }
+            }, seconds(2));
+        }
+
+        this.retrieveStateDebouncers[rd.device.ieeeAddr]?.();
+    }
+}
+
+module.exports = AvailabilityNew;

lib/extension/bind.js

@@ -1,118 +1,422 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+const debounce = require('debounce');
+const zigbeeHersdman = require('zigbee-herdsman');
 
-const clusters = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
+    'hvacThermostat', 'msTemperatureMeasurement'];
 
-// See zigbee-herdsman-converters devices.js
+// See zigbee-herdsman-converters
 const defaultBindGroup = {type: 'group_number', ID: 901};
 
+const defaultReportConfiguration = {
+    minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: 1,
+};
+
+const getColorCapabilities = async (endpoint) => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') == null) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const reportClusters = {
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultReportConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultReportConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultReportConfiguration,
+            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultReportConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultReportConfiguration},
+    ],
+};
+
+const pollOnMessage = [
+    {
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genLevelCtrl: [
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+                {type: 'commandToggle', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+];
+
 class Bind extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-        }
+        this.eventBus.on(`groupMembersChanged`, (d) => this.groupMembersChanged(d), this.constructor.name);
+        this.pollDebouncers = {};
     }
 
     parseMQTTMessage(topic, message) {
         let type = null;
         let sourceKey = null;
         let targetKey = null;
+        let clusters = null;
+        let skipDisableReporting = false;
 
         if (this.legacyApi && topic.match(legacyTopicRegex)) {
             topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
             type = topic.split('/')[0];
             sourceKey = topic.replace(`${type}/`, '');
             targetKey = message;
+        } else if (topic.match(topicRegex)) {
+            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
+            message = JSON.parse(message);
+            sourceKey = message.from;
+            targetKey = message.to;
+            clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
         }
 
-        return {type, sourceKey, targetKey};
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
     }
 
     async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey} = this.parseMQTTMessage(topic, message);
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(topic, message);
         if (!type) return null;
+        message = utils.parseJSON(message, message);
 
-        // Find source; can only be a device and target
+        let error = null;
         const source = this.zigbee.resolveEntity(sourceKey);
-        assert(source != null && source.type === 'device', 'Source undefined or not a device');
         const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
-        assert(target != null, 'Target is unknown');
+        const responseData = {from: sourceKey, to: targetKey};
 
-        const sourceName = source.settings.friendlyName;
-        const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
-        let attemptedToBindSomething = false;
+        if (!source || source.type !== 'device') {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const sourceName = source.settings.friendlyName;
+            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
+            const successfulClusters = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
 
-        // Find which clusters are supported by both the source and target.
-        // Groups are assumed to support all clusters.
-        for (const cluster of clusters) {
-            const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
+            let bindTarget = null;
+            if (target.type === 'group') bindTarget = target.group;
+            else if (target.type === 'group_number') bindTarget = target.ID;
+            else bindTarget = target.endpoint;
 
-            if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                attemptedToBindSomething = true;
-                try {
-                    let bindTarget = null;
-                    if (target.type === 'group') bindTarget = target.group;
-                    else if (target.type === 'group_number') bindTarget = target.ID;
-                    else bindTarget = target.endpoint;
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusterCandidates) {
+                if (clusters && !clusters.includes(cluster)) continue;
+                let matchingClusters = false;
 
-                    if (type === 'bind') {
-                        await source.endpoint.bind(cluster, bindTarget);
-                    } else {
-                        await source.endpoint.unbind(cluster, bindTarget);
-                    }
+                const anyClusterValid = target.type === 'group' || target.type === 'group_number' ||
+                    target.device.type === 'Coordinator';
 
-                    logger.info(
-                        `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                        `'${sourceName}' to '${targetName}'`,
-                    );
+                if (!anyClusterValid) {
+                    matchingClusters = ((target.endpoint.supportsInputCluster(cluster) &&
+                            source.endpoint.supportsOutputCluster(cluster)) ||
+                            (source.endpoint.supportsInputCluster(cluster) &&
+                            target.endpoint.supportsOutputCluster(cluster)) );
+                }
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                const sourceValid = source.endpoint.supportsInputCluster(cluster) ||
+                source.endpoint.supportsOutputCluster(cluster);
+
+                if ( sourceValid && (anyClusterValid || matchingClusters)) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
+                    attemptedClusters.push(cluster);
+
+                    try {
+                        if (type === 'bind') {
+                            await source.endpoint.bind(cluster, bindTarget);
+                        } else {
+                            await source.endpoint.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${sourceName}' to '${targetName}'`,
                         );
-                    }
-                } catch (error) {
-                    logger.error(
-                        `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                        `'${targetName}' (${error})`,
-                    );
 
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        this.mqtt.publish(
-                            'bridge/log',
-                            JSON.stringify({type: `device_${type}_failed`,
-                                message: {from: sourceName, to: targetName, cluster}}),
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
+                            `'${targetName}' (${error})`,
                         );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}_failed`,
+                                    message: {from: sourceName, to: targetName, cluster}}),
+                            );
+                        }
                     }
                 }
             }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
+
+            if (successfulClusters.length !== 0) {
+                if (type === 'bind') {
+                    await this.setupReporting(source.endpoint.binds.filter((b) =>
+                        successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
+                } else if ((target.type !== 'group_number') && !skipDisableReporting) {
+                    await this.disableUnnecessaryReportings(bindTarget);
+                }
+            }
         }
 
-        if (!attemptedToBindSomething) {
-            logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
+        }
 
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
-                );
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit(`devicesChanged`);
+        }
+    }
+
+    async groupMembersChanged(data) {
+        if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.getClients().map((c) => c.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.group);
+            await this.setupReporting(bindsToGroup);
+        } else { // action === remove/remove_all
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
+        }
+    }
+
+    getSetupReportingEndpoints(bind, coordinatorEp) {
+        const endpoints = bind.target.constructor.name === 'Group' ? bind.target.members : [bind.target];
+        return endpoints.filter((e) => {
+            const supportsInputCluster = e.supportsInputCluster(bind.cluster.name);
+            const hasConfiguredReporting = !!e.configuredReportings.find((c) => c.cluster.name === bind.cluster.name);
+            const hasBind = !!e.binds.find((b) => b.cluster.name === bind.cluster.name && b.target === coordinatorEp);
+            return supportsInputCluster && !(hasBind && hasConfiguredReporting);
+        });
+    }
+
+    async setupReporting(binds) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
+            for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
+                const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+                try {
+                    await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
+
+                    const items = await reportClusters[bind.cluster.name]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(bind.cluster.name, items);
+                    logger.info(`Succesfully setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                } catch (error) {
+                    logger.warn(`Failed to setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                }
+            }
+        }
+
+        this.eventBus.emit(`devicesChanged`);
+    }
+
+    async disableUnnecessaryReportings(target) {
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        const endpoints = target.constructor.name === 'Group' ? target.members : [target];
+        for (const endpoint of endpoints) {
+            const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+            const boundClusters = endpoint.binds.filter((b) => b.target === coordinatorEndpoint)
+                .map((b) => b.cluster.name);
+            const requiredClusters = this.zigbee.getClients().map((c) => c.endpoints).reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
+                    if (bind.target.constructor.name === 'Group') {
+                        return bind.target.members.includes(endpoint);
+                    } else {
+                        return bind.target === endpoint;
+                    }
+                }).map((b) => b.cluster.name).filter((v, i, a) => a.indexOf(v) === i);
+
+            for (const cluster of boundClusters.filter((c) => !requiredClusters.includes(c) && c in reportClusters)) {
+                try {
+                    await endpoint.unbind(cluster, coordinatorEndpoint);
+                    const items = await reportClusters[cluster]
+                        .filter(async (a) => !a.condition || await a.condition(endpoint))
+                        .map((a) => {
+                            const result = {...a, maximumReportInterval: 0xFFFF};
+                            delete result.condition;
+                            return result;
+                        });
+                    await endpoint.configureReporting(cluster, items);
+                    logger.info(`Succesfully disabled reporting for '${entity}' cluster '${cluster}'`);
+                } catch (error) {
+                    logger.warn(`Failed to disable reporting for '${entity}' cluster '${cluster}'`);
+                }
+            }
+
+            this.eventBus.emit('reportingDisabled', {device: endpoint.getDevice()});
+        }
+    }
+
+    async onZigbeeEvent(type, data, resolvedEntity) {
+        if (type === 'message') {
+            this.poll(data);
+        }
+    }
+
+    poll(message) {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
+            utils.equalsPartial(message.data, c.data)),
+        );
+
+        if (polls.length) {
+            let toPoll = [];
+
+            // Add bound devices
+            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
+                e.binds.map((e) => e).filter((e) => e.target))));
+            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
+            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
+            toPoll = toPoll.map((e) => e.target);
+
+            // If message is published to a group, add members of the group
+            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
+            if (group) {
+                toPoll = toPoll.concat(group.members);
+            }
+
+            toPoll = new Set(toPoll);
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID)) {
+                        continue;
+                    }
+
+                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, poll.read.attributes);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
             }
         }
     }

lib/extension/bridge.js

@@ -3,70 +3,148 @@ const utils = require('../util/utils');
 const Extension = require('./extension');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const settings = require('../util/settings');
+const Transport = require('winston-transport');
+const stringify = require('json-stable-stringify-without-jsonify');
+const objectAssignDeep = require(`object-assign-deep`);
+const {detailedDiff} = require('deep-object-diff');
 
 const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
 
 class Bridge extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
-
+        this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
+        this.lastJoinedDeviceIeeeAddr = null;
+        this.setupMQTTLogging();
+        this.restartRequired = false;
 
         this.requestLookup = {
-            'permitjoin': this.permitJoin.bind(this),
-            // 'device/remove': this.deviceRemove.bind(this),
-            // 'device/forceremove': this.deviceForceRemove.bind(this),
-            // 'device/ban': this.deviceBan.bind(this),
-            // 'group/remove': this.groupRemove.bind(this),
+            'device/options': this.deviceOptions.bind(this),
+            'device/configure_reporting': this.deviceConfigureReporting.bind(this),
+            'device/remove': this.deviceRemove.bind(this),
+            'device/rename': this.deviceRename.bind(this),
+            'group/add': this.groupAdd.bind(this),
+            'group/options': this.groupOptions.bind(this),
+            'group/remove': this.groupRemove.bind(this),
+            'group/rename': this.groupRename.bind(this),
+            'permit_join': this.permitJoin.bind(this),
+            'restart': this.restart.bind(this),
+            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
+            'touchlink/identify': this.touchlinkIdentify.bind(this),
+            'touchlink/scan': this.touchlinkScan.bind(this),
+            'health_check': this.healthCheck.bind(this),
+            'options': this.bridgeOptions.bind(this),
+            // Below are deprecated
+            'config/last_seen': this.configLastSeen.bind(this),
+            'config/homeassistant': this.configHomeAssistant.bind(this),
+            'config/elapsed': this.configElapsed.bind(this),
+            'config/log_level': this.configLogLevel.bind(this),
         };
     }
 
     async onMQTTConnected() {
-        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
+        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
         this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
 
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
+        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups(), this.constructor.name);
+        this.eventBus.on(`devicesChanged`, () => {
+            this.publishDevices();
+            this.publishInfo();
+        }, this.constructor.name);
+        this.eventBus.on(`deviceRenamed`, () => {
+            this.publishInfo();
+        }, this.constructor.name);
+        this.eventBus.on(`groupRenamed`, () => {
+            this.publishInfo();
+        }, this.constructor.name);
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+        this.permitJoinChanged = this.permitJoinChanged.bind(this);
+        this.zigbee.on('permitJoinChanged', this.permitJoinChanged);
         await this.publishInfo();
         await this.publishDevices();
         await this.publishGroups();
     }
 
+    async stop() {
+        super.stop();
+        this.zigbee.removeListener('permitJoinChanged', this.permitJoinChanged);
+    }
+
+    setupMQTTLogging() {
+        const mqtt = this.mqtt;
+        class EventTransport extends Transport {
+            log(info, callback) {
+                const payload = stringify({message: info.message, level: info.level});
+                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
+                callback();
+            }
+        }
+
+        logger.addTransport(new EventTransport());
+    }
+
+    permitJoinChanged(data) {
+        if (!this.zigbee.isStopping()) {
+            this.publishInfo();
+        }
+    }
+
     async onMQTTMessage(topic, message) {
         const match = topic.match(requestRegex);
         if (match && this.requestLookup[match[1].toLowerCase()]) {
-            message = JSON.parse(message);
+            message = utils.parseJSON(message, message);
+
             try {
                 const response = await this.requestLookup[match[1].toLowerCase()](message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             } catch (error) {
                 logger.error(`Request '${topic}' failed with error: '${error.message}'`);
                 const response = utils.getResponse(message, {}, error.message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, JSON.stringify(response));
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
             }
         }
     }
 
     async onZigbeeEvent(type, data, resolvedEntity) {
-        if (['deviceJoined', 'deviceLeave', 'deviceInterview'].includes(type)) {
-            let payload;
-            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
-            if (type === 'deviceJoined') payload = {friendlyName: resolvedEntity.settings.friendlyName, ieeeAddress};
-            else if (type === 'deviceInterview') {
-                payload = {friendlyName: resolvedEntity.settings.friendlyName, status: data.status, ieeeAddress};
-                if (data.status === 'successful') {
-                    payload.supported = !!resolvedEntity.definition;
-                    payload.definition = resolvedEntity.definition ? {
-                        model: resolvedEntity.definition.model,
-                        vendor: resolvedEntity.definition.vendor,
-                        description: resolvedEntity.definition.description,
-                        supports: resolvedEntity.definition.supports,
-                    } : null;
-                }
-            } else payload = {ieeeAddress}; // deviceLeave
-
-            await this.mqtt.publish('bridge/event', JSON.stringify({type, data: payload}), {retain: false, qos: 0});
+        if (type === 'deviceJoined' && resolvedEntity) {
+            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
         }
 
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
+        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
+            let payload;
+            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
+            if (type === 'deviceJoined') {
+                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
+            } else if (type === 'deviceInterview') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
+                };
+                if (data.status === 'successful') {
+                    const definition = resolvedEntity.definition;
+                    payload.supported = !!definition;
+                    payload.definition = this.getDefinitionPayload(definition, resolvedEntity);
+                }
+            } else if (type === 'deviceAnnounce') {
+                payload = {
+                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
+                };
+            } else {
+                // deviceLeave
+                const settingsDevice = settings.getDevice(ieeeAddress);
+                payload = {ieee_address: ieeeAddress,
+                    friendly_name: settingsDevice ? settingsDevice.friendlyName : null};
+            }
+
+            await this.mqtt.publish(
+                'bridge/event',
+                stringify({type: utils.toSnakeCase(type), data: payload}),
+                {retain: false, qos: 0},
+            );
+        }
+
+        if (['deviceLeave', 'deviceInterview', 'deviceNetworkAddressChanged',
+            'deviceJoined', 'deviceAnnounce'].includes(type)) {
             await this.publishDevices();
         }
     }
@@ -75,136 +153,560 @@ class Bridge extends Extension {
      * Requests
      */
 
-    // async deviceRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('remove', 'device', message);
-    // }
+    async deviceOptions(message) {
+        return this.changeEntityOptions('device', message);
+    }
 
-    // async deviceForceRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('force_remove', 'device', message);
-    // }
+    async groupOptions(message) {
+        return this.changeEntityOptions('group', message);
+    }
 
-    // async deviceBan(message) {
-    //     return this.removeForceRemoveOrBanEntity('ban', 'device', message);
-    // }
+    async bridgeOptions(message) {
+        if (typeof message !== 'object' || typeof message.options !== 'object') {
+            throw new Error(`Invalid payload`);
+        }
 
-    // async groupRemove(message) {
-    //     return this.removeForceRemoveOrBanEntity('remove', 'group', message);
-    // }
+        const diff = detailedDiff(settings.get(), message.options);
+
+        // Remove any settings that are in the deleted.diff but not in the passed options
+        const cleanupDeleted = (options, deleted) => {
+            for (const key of Object.keys(deleted)) {
+                if (!(key in options)) {
+                    delete deleted[key];
+                } else if (!Array.isArray(options[key])) {
+                    cleanupDeleted(options[key], deleted[key]);
+                }
+            }
+        };
+        cleanupDeleted(message.options, diff.deleted);
+
+        const newSettings = objectAssignDeep.noMutate(diff.added, diff.updated, diff.deleted);
+
+        // deep-object-diff converts arrays to objects, set original array back here
+        const convertBackArray = (before, after) => {
+            for (const [key, afterValue] of Object.entries(after)) {
+                const beforeValue = before[key];
+                if (Array.isArray(beforeValue)) {
+                    after[key] = beforeValue;
+                } else if (afterValue && typeof beforeValue === 'object') {
+                    convertBackArray(beforeValue, afterValue);
+                }
+            }
+        };
+        convertBackArray(message.options, newSettings);
+
+        const restartRequired = settings.apply(newSettings);
+        if (restartRequired) this.restartRequired = true;
+
+        // Apply some settings on-the-fly.
+        if (newSettings.hasOwnProperty('permit_join')) {
+            await this.zigbee.permitJoin(newSettings.permit_join);
+        }
+
+        if (newSettings.hasOwnProperty('homeassistant')) {
+            this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+        }
+
+        if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            logger.setLevel(newSettings.advanced.log_level);
+        }
+
+        logger.info('Succesfully changed options');
+        this.publishInfo();
+        return utils.getResponse(message, {restart_required: this.restartRequired}, null);
+    }
+
+    async deviceRemove(message) {
+        return this.removeEntity('device', message);
+    }
+
+    async groupRemove(message) {
+        return this.removeEntity('group', message);
+    }
+
+    async healthCheck(message) {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    async groupAdd(message) {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
+    }
+
+    async deviceRename(message) {
+        return this.renameEntity('device', message);
+    }
+
+    async groupRename(message) {
+        return this.renameEntity('group', message);
+    }
+
+    async restart(message) {
+        // Wait 500 ms before restarting so response can be send.
+        setTimeout(this.restartCallback, 500);
+        logger.info('Restarting Zigbee2MQTT');
+        return utils.getResponse(message, {}, null);
+    }
 
     async permitJoin(message) {
-        const value = typeof message === 'object' ? message.value : message;
-        await this.zigbee.permitJoin(value);
-        await this.publishInfo();
-        return utils.getResponse(message, {value: value}, null);
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value;
+        let time;
+        let resolvedEntity;
+        if (typeof message === 'object') {
+            value = message.value;
+            time = message.time;
+            if (message.device) {
+                resolvedEntity = this.zigbee.resolveEntity(message.device);
+                if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        if (typeof value === 'string') {
+            value = value.toLowerCase() === 'true';
+        }
+
+        await this.zigbee.permitJoin(value, resolvedEntity, time);
+        const response = {value};
+        if (resolvedEntity) response.device = message.device;
+        if (time) response.time = message.time;
+        return utils.getResponse(message, response, null);
+    }
+
+    // Deprecated
+    configLastSeen(message) {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configHomeAssistant(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        this.enableDisableExtension(value, 'HomeAssistant');
+        settings.set(['homeassistant'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configElapsed(message) {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    configLogLevel(message) {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message);
+        if (!allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    async touchlinkIdentify(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
+    async touchlinkFactoryReset(message) {
+        let result = false;
+        const payload = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, payload, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    async touchlinkScan(message) {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
     }
 
     /**
      * Utils
      */
 
-    // async removeForceRemoveOrBanEntity(action, entityType, message) {
-    //     const ID = typeof message === 'object' ? message.ID : message.trim();
-    //     const entity = this.zigbee.resolveEntity(ID);
-    //     if (!entity || entity.type !== entityType) {
-    //         throw new Error(`${ID} is not a ${entityType}`);
-    //     }
+    getValue(message) {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
 
-    //     const lookup = {
-    //         ban: ['banned', 'Banning', 'ban'],
-    //         force_remove: ['force_removed', 'Force removing', 'force remove'],
-    //         remove: ['removed', 'Removing', 'remove'],
-    //     };
+            return message.value;
+        } else {
+            return message;
+        }
+    }
 
-    //     try {
-    //         logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
-    //         if (entity.type === 'device') {
-    //             if (action === 'ban') {
-    //                 settings.banDevice(entity.settings.ID);
-    //             }
+    async changeEntityOptions(entityType, message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
 
-    //             action === 'force_remove' ?
-    //                 await entity.device.removeFromDatabase() : await entity.device.removeFromNetwork();
-    //         } else {
-    //             await entity.group.removeFromDatabase();
-    //         }
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        settings.changeEntityOptions(ID, message.options);
+        const cleanup = (o) => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+        const oldOptions = cleanup(entity.settings);
+        const newOptions = cleanup(settings.getEntity(ID));
+        await this.publishInfo();
 
-    //         // Fire event
-    //         if (entity.type === 'device') {
-    //             this.eventBus.emit('deviceRemoved', {device: entity.device});
-    //         }
+        logger.info(`Changed config for ${entityType} ${ID}`);
 
-    //         // Remove from configuration.yaml
-    //         entity.type === 'device' ?
-    //             settings.removeDevice(entity.settings.ID) : settings.removeGroup(entity.settings.ID);
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
 
-    //         // Remove from state
-    //         this.state.remove(entity.settings.ID);
+    async deviceConfigureReporting(message) {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
+            !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
+            !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
+            throw new Error(`Invalid payload`);
+        }
 
-    //         logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-    //         entity.type === 'device' ? this.publishDevices() : this.publishGroups();
-    //         return utils.getResponse(message, {ID}, null);
-    //     } catch (error) {
-    //         throw new Error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
-    //     }
-    // }
+        const ID = message.id;
+        const endpoint = this.getEntity('device', ID).endpoint;
+
+        const coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
+        const coordinatorEndpoint = coordinator.getEndpoint(1);
+        await endpoint.bind(message.cluster, coordinatorEndpoint);
+
+        await endpoint.configureReporting(message.cluster, [{
+            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
+            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
+        }]);
+
+        this.publishDevices();
+
+        logger.info(`Configured reporting for '${message.id}', '${message.cluster}.${message.attribute}'`);
+
+        return utils.getResponse(message, {
+            id: message.id, cluster: message.cluster, maximum_report_interval: message.maximum_report_interval,
+            minimum_report_interval: message.minimum_report_interval, reportable_change: message.reportable_change,
+            attribute: message.attribute,
+        }, null);
+    }
+
+    renameEntity(entityType, message) {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
+        const entity = this.getEntity(entityType, from);
+
+        settings.changeFriendlyName(from, to);
+
+        // Clear retained messages
+        this.mqtt.publish(entity.name, '', {retain: true});
+
+        const oldFriendlyName = entity.settings.friendlyName;
+        if (entity.type === 'device') {
+            this.publishDevices();
+            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename, from: oldFriendlyName, to});
+        } else {
+            this.publishGroups();
+            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename, from: oldFriendlyName, to});
+        }
+
+        // Repulish entity state
+        this.publishEntityState(to, {});
+
+        return utils.getResponse(
+            message,
+            {from: oldFriendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
+    }
+
+    async removeEntity(entityType, message) {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+            if (entity.type === 'device') {
+                if (block) {
+                    settings.blockDevice(entity.settings.ID);
+                }
+
+                if (force) {
+                    await entity.device.removeFromDatabase();
+                } else {
+                    await entity.device.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    await entity.group.removeFromDatabase();
+                } else {
+                    await entity.group.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity.type === 'device') {
+                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+            }
+
+            // Remove from configuration.yaml
+            if (entity.type === 'device') {
+                settings.removeDevice(entity.settings.ID);
+            } else {
+                settings.removeGroup(entity.settings.ID);
+            }
+
+            // Remove from state
+            this.state.remove(entity.settings.ID);
+
+            // Clear any retained messages
+            this.mqtt.publish(entity.name, '', {retain: true});
+
+            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
+
+            if (entity.type === 'device') {
+                this.publishGroups();
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type, ID) {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.type !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
 
     async publishInfo() {
+        const config = objectAssignDeep.noMutate({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
         const payload = {
             version: this.zigbee2mqttVersion.version,
             commit: this.zigbee2mqttVersion.commitHash,
             coordinator: this.coordinatorVersion,
-            logLevel: logger.getLevel(),
-            permitJoin: await this.zigbee.getPermitJoin(),
+            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            log_level: logger.getLevel(),
+            permit_join: this.zigbee.getPermitJoin(),
+            permit_join_timeout: this.zigbee.getPermitJoinTimeout(),
+            restart_required: this.restartRequired,
+            config,
+            config_schema: settings.schema,
         };
 
-        await this.mqtt.publish('bridge/info', JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    async publishDevices(topic, message) {
-        const devices = this.zigbee.getClients().map((device) => {
+    async publishDevices() {
+        const devices = this.zigbee.getDevices().map((device) => {
             const definition = zigbeeHerdsmanConverters.findByDevice(device);
             const resolved = this.zigbee.resolveEntity(device);
-            const definitionPayload = definition ? {
-                model: definition.model,
-                vendor: definition.vendor,
-                description: definition.description,
-                supports: definition.supports,
-            } : null;
+            const endpoints = {};
+            for (const endpoint of device.endpoints) {
+                const data = {
+                    bindings: [],
+                    configured_reportings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    let target;
+
+                    if (bind.target.constructor.name === 'Endpoint') {
+                        target = {
+                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
+                        };
+                    } else {
+                        target = {type: 'group', id: bind.target.groupID};
+                    }
+
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                for (const configuredReporting of endpoint.configuredReportings) {
+                    data.configured_reportings.push({
+                        cluster: configuredReporting.cluster.name,
+                        attribute: configuredReporting.attribute.name || configuredReporting.attribute.ID,
+                        minimum_report_interval: configuredReporting.minimumReportInterval,
+                        maximum_report_interval: configuredReporting.maximumReportInterval,
+                        reportable_change: configuredReporting.reportableChange,
+                    });
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
 
             return {
-                ieeeAddress: device.ieeeAddr,
+                ieee_address: device.ieeeAddr,
                 type: device.type,
-                networkAddress: device.networkAddress,
+                network_address: device.networkAddress,
                 supported: !!definition,
-                friendlyName: resolved.name,
-                definition: definitionPayload,
-                powerSource: device.powerSource,
-                softwareBuildID: device.softwareBuildID,
-                dateCode: device.dateCode,
+                friendly_name: resolved.name,
+                definition: this.getDefinitionPayload(definition, resolved),
+                power_source: device.powerSource,
+                software_build_id: device.softwareBuildID,
+                date_code: device.dateCode,
+                model_id: device.modelID,
                 interviewing: device.interviewing,
-                interviewCompleted: device.interviewCompleted,
+                interview_completed: device.interviewCompleted,
+                manufacturer: device.manufacturerName,
+                endpoints,
             };
         });
 
-        await this.mqtt.publish('bridge/devices', JSON.stringify(devices), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/devices', stringify(devices), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
     }
 
-    async publishGroups(topic, message) {
+    async publishGroups() {
         const groups = this.zigbee.getGroups().map((group) => {
             const resolved = this.zigbee.resolveEntity(group);
             return {
-                ID: group.groupID,
-                friendlyName: resolved.name,
+                id: group.groupID,
+                friendly_name: group.groupID === 901 ? 'default_bind_group' : resolved.name,
                 members: group.members.map((m) => {
                     return {
-                        ieeeAddress: m.deviceIeeeAddress,
+                        ieee_address: m.deviceIeeeAddress,
                         endpoint: m.ID,
                     };
                 }),
             };
         });
 
-        await this.mqtt.publish('bridge/groups', JSON.stringify(groups), {retain: true, qos: 0});
+        await this.mqtt.publish(
+            'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    getDefinitionPayload(definition, resolvedDevice) {
+        if (definition) {
+            let icon = null;
+            if (resolvedDevice && resolvedDevice.hasOwnProperty('settings')) {
+                icon = resolvedDevice.settings.icon;
+            }
+            if (!icon) {
+                icon = definition.icon;
+            }
+            if (icon) {
+                let zigbeeModel = '';
+                /* istanbul ignore else */
+                if (resolvedDevice && resolvedDevice.hasOwnProperty('device')) {
+                    zigbeeModel = resolvedDevice.device.modelID;
+                }
+                icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(zigbeeModel));
+                icon = icon.replace('${model}', utils.sanitizeImageParameter(definition.model));
+            }
+            return {
+                model: definition.model,
+                vendor: definition.vendor,
+                description: definition.description,
+                exposes: definition.exposes,
+                supports_ota: !!definition.ota,
+                icon,
+            };
+        } else {
+            return null;
+        }
     }
 }
 

lib/extension/configure.js

@@ -1,6 +1,9 @@
 const settings = require('../util/settings');
+const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
+const zhc = require('zigbee-herdsman-converters');
 
 /**
  * This extension calls the zigbee-herdsman-converters definition configure() method
@@ -10,20 +13,37 @@ class Configure extends Extension {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
 
         this.configuring = new Set();
+        this.onReportingDisabled = this.onReportingDisabled.bind(this);
         this.attempts = {};
 
-        this.legacyApi = settings.get().advanced.legacy_api;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled, this.constructor.name);
     }
 
-    shouldConfigure(resolvedEntity) {
+    onReportingDisabled(data) {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        const device = data.device;
+
+        const resolvedEntity = this.zigbee.resolveEntity(device);
+        if (resolvedEntity.device.meta && resolvedEntity.device.meta.hasOwnProperty('configured')) {
+            delete device.meta.configured;
+            device.save();
+        }
+
+        if (this.shouldConfigure(resolvedEntity)) {
+            this.configure(resolvedEntity, 'reporting_disabled');
+        }
+    }
+
+    shouldConfigure(resolvedEntity, event) {
         if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
             return false;
         }
 
         if (resolvedEntity.device.meta &&
             resolvedEntity.device.meta.hasOwnProperty('configured') &&
-            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
+            resolvedEntity.device.meta.configured === zhc.getConfigureKey(resolvedEntity.definition)) {
             return false;
         }
 
@@ -31,23 +51,16 @@ class Configure extends Extension {
             return false;
         }
 
+        // Only configure end devices when a message is received, otherwise it will likely fails as they are sleeping.
+        if (resolvedEntity.device.type === 'EndDevice' && event !== 'message_received') {
+            return false;
+        }
+
         return true;
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            if (topic !== this.legacyTopic) {
-                return;
-            }
-
+        if (topic === this.legacyTopic) {
             const resolvedEntity = this.zigbee.resolveEntity(message);
             if (!resolvedEntity || resolvedEntity.type !== 'device') {
                 logger.error(`Device '${message}' does not exist`);
@@ -60,6 +73,26 @@ class Configure extends Extension {
             }
 
             this.configure(resolvedEntity, true);
+        } else if (topic === this.topic) {
+            message = utils.parseJSON(message, message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const resolvedEntity = this.zigbee.resolveEntity(ID);
+            if (!resolvedEntity || resolvedEntity.type !== 'device') {
+                error = `Device '${ID}' does not exist`;
+            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
+                error = `Device '${resolvedEntity.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(resolvedEntity, true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
         }
     }
 
@@ -68,7 +101,7 @@ class Configure extends Extension {
 
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.shouldConfigure(resolvedEntity)) {
+            if (this.shouldConfigure(resolvedEntity, 'started')) {
                 await this.configure(resolvedEntity);
             }
         }
@@ -82,12 +115,12 @@ class Configure extends Extension {
             device.save();
         }
 
-        if (this.shouldConfigure(resolvedEntity)) {
+        if (this.shouldConfigure(resolvedEntity, 'message_received')) {
             this.configure(resolvedEntity);
         }
     }
 
-    async configure(resolvedEntity, force=false) {
+    async configure(resolvedEntity, force=false, thowError=false) {
         const device = resolvedEntity.device;
         if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
             return false;
@@ -101,16 +134,20 @@ class Configure extends Extension {
 
         logger.info(`Configuring '${resolvedEntity.name}'`);
         try {
-            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
+            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint, logger);
             logger.info(`Successfully configured '${resolvedEntity.name}'`);
-            device.meta.configured = resolvedEntity.definition.meta.configureKey;
+            device.meta.configured = zhc.getConfigureKey(resolvedEntity.definition);
             device.save();
+            this.eventBus.emit(`devicesChanged`);
         } catch (error) {
-            logger.error(
-                `Failed to configure '${resolvedEntity.name}', ` +
-                `attempt ${this.attempts[device.ieeeAddr] + 1} (${error.stack})`,
-            );
             this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
         }
 
         this.configuring.delete(device.ieeeAddr);

lib/extension/extension.js

@@ -46,7 +46,9 @@ class Extension {
     /**
      * Is called once the extension has to stop
      */
-    // stop() {}
+    stop() {
+        this.eventBus.removeListenersExtension(this.constructor.name);
+    }
 }
 
 module.exports = Extension;

lib/extension/extensionts.ts

@@ -0,0 +1,69 @@
+abstract class ExtensionTS {
+    protected zigbee: TempZigbee;
+    protected mqtt: TempMQTT;
+    protected state: TempState;
+    protected publishEntityState: TempPublishEntityState;
+    protected eventBus: TempEventBus;
+
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     */
+    constructor(zigbee: TempZigbee, mqtt: TempMQTT, state: TempState,
+        publishEntityState: TempPublishEntityState, eventBus: TempEventBus) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+    }
+
+    /**
+     * This method is called by the controller once Zigbee has been started.
+     */
+    /* istanbul ignore next */
+    onZigbeeStarted(): void {}
+
+    /**
+     * This method is called by the controller once connected to the MQTT server.
+     */
+    /* istanbul ignore next */
+    onMQTTConnected(): void {}
+
+    /**
+     * Is called when a Zigbee message from a device is received.
+     * @param {string} type Type of the message
+     * @param {Object} data Data of the message
+     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
+     * @param {Object?} settingsDevice Device settings
+     */
+    /* istanbul ignore next */
+    /* eslint-disable-next-line @typescript-eslint/no-unused-vars */
+    onZigbeeEvent(type: ZigbeeEventType, data: ZigbeeEventData, resolvedEntity: ResolvedEntity): void {}
+
+    /**
+     * Is called when a MQTT message is received
+     * @param {string} topic Topic on which the message was received
+     * @param {Object} message The received message
+     * @return {boolean} if the message was handled
+     */
+    /* istanbul ignore next */
+    /* eslint-disable-next-line @typescript-eslint/no-unused-vars */
+    onMQTTMessage(topic: string, message: string): boolean {
+        return false;
+    }
+
+    /**
+     * Is called once the extension has to stop
+     */
+    stop(): void {
+        this.eventBus.removeListenersExtension(this.constructor.name);
+    }
+}
+
+export default ExtensionTS;

lib/extension/externalConverters.js

@@ -0,0 +1,18 @@
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+
+class ExternalConverters extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
+        }
+    }
+}
+
+module.exports = ExternalConverters;

lib/extension/externalExtension.js

@@ -0,0 +1,114 @@
+const settings = require('../util/settings');
+const Extension = require('./extension');
+const utils = require('../util/utils');
+const fs = require('fs');
+const data = require('./../util/data');
+const path = require('path');
+const logger = require('./../util/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
+
+class ExternalExtension extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus,
+        enableDisableExtension, restartCallback, addExtension) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.args = [zigbee, mqtt, state, publishEntityState, eventBus];
+        this.addExtension = addExtension;
+        this.enableDisableExtension = enableDisableExtension;
+        this.extensionsBaseDir = 'extension';
+        this.loadExtension = this.loadExtension.bind(this);
+        this.requestLookup = {
+            'save': this.saveExtension.bind(this),
+            'remove': this.removeExtension.bind(this),
+        };
+        this.loadUserDefinedExtensions();
+    }
+    getExtensionsBasePath() {
+        return data.joinPath(this.extensionsBaseDir);
+    }
+
+    getListOfUserDefinedExtensions() {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
+                const extensonFilePath = path.join(basePath, fileName);
+                return {'name': fileName, 'code': fs.readFileSync(extensonFilePath, 'utf-8')};
+            });
+        } else {
+            return [];
+        }
+    }
+    removeExtension(message) {
+        const {name} = message;
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionToBeRemoved = extensions.find((e) => e.name === name);
+
+        if (extensionToBeRemoved) {
+            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            const basePath = this.getExtensionsBasePath();
+            const extensonFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensonFilePath);
+            this.publishExtensions();
+            logger.info(`Extension ${name} removed`);
+            return utils.getResponse(message, {}, null);
+        } else {
+            return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
+        }
+    }
+    saveExtension(message) {
+        const {name, code} = message;
+        const ModuleConstructor = utils.loadModuleFromText(code);
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, path.basename(name));
+        fs.writeFileSync(extensonFilePath, code);
+        this.publishExtensions();
+        logger.info(`Extension ${name} loaded`);
+        return utils.getResponse(message, {}, null);
+    }
+
+
+    async onMQTTMessage(topic, message) {
+        const match = topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            message = utils.parseJSON(message, message);
+            try {
+                const response = await this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    loadExtension(ConstructorClass) {
+        this.enableDisableExtension(false, ConstructorClass.name);
+        this.addExtension(new ConstructorClass(...this.args, settings, logger));
+    }
+
+    loadUserDefinedExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        extensions
+            .map(({code}) => utils.loadModuleFromText(code))
+            .map(this.loadExtension);
+    }
+    async onMQTTConnected() {
+        this.publishExtensions();
+    }
+
+    async publishExtensions() {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}
+
+module.exports = ExternalExtension;

lib/extension/frontend.js

@@ -0,0 +1,129 @@
+const http = require('http');
+const serveStatic = require('serve-static');
+const finalhandler = require('finalhandler');
+const Extension = require('./extension');
+const logger = require('../util/logger');
+const frontend = require('zigbee2mqtt-frontend');
+const WebSocket = require('ws');
+const url = require('url');
+const settings = require('../util/settings');
+const utils = require('../util/utils');
+const stringify = require('json-stable-stringify-without-jsonify');
+
+/**
+ * This extension servers the frontend
+ */
+class Frontend extends Extension {
+    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus);
+        this.onRequest = this.onRequest.bind(this);
+        this.onUpgrade = this.onUpgrade.bind(this);
+        this.mqttBaseTopic = settings.get().mqtt.base_topic;
+        this.onMQTTPublishMessage = this.onMQTTPublishMessage.bind(this);
+        this.mqtt.on('publishMessage', this.onMQTTPublishMessage);
+        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+        this.host = settings.get().frontend.host || '0.0.0.0';
+        this.port = settings.get().frontend.port || 8080;
+        this.authToken = settings.get().frontend.auth_token || false;
+        this.retainedMessages = new Map();
+        /* istanbul ignore next */
+        const options = {setHeaders: (res, path) => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+    }
+
+    onZigbeeStarted() {
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    async stop() {
+        for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
+            client.terminate();
+        }
+        this.wss.close();
+        return new Promise((resolve) => {
+            this.server.close(resolve);
+        });
+    }
+    onRequest(request, response) {
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+    authenticate(request, cb) {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    onUpgrade(request, socket, head) {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    onWebSocketConnection(ws) {
+        ws.on('message', (data, isBinary) => {
+            if (!isBinary && data) {
+                const message = data.toString();
+                const {topic, payload} = utils.parseJSON(message, message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.getClients()) {
+            let payload = {};
+            const resolvedEntity = this.zigbee.resolveEntity(device);
+            if (this.state.exists(device.ieeeAddr)) {
+                payload = {...payload, ...this.state.get(device.ieeeAddr)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
+            }
+
+            if (resolvedEntity.device.linkquality !== undefined) {
+                payload.linkquality = resolvedEntity.device.linkquality;
+            }
+
+            ws.send(stringify({topic: resolvedEntity.name, payload}));
+        }
+    }
+
+    onMQTTPublishMessage(data) {
+        let {topic, payload, options} = data;
+        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            topic = topic.substring(this.mqttBaseTopic.length + 1);
+            payload = utils.parseJSON(payload, payload);
+            if (options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            for (const client of this.wss.clients) {
+                if (client.readyState === WebSocket.OPEN) {
+                    client.send(stringify({topic, payload}));
+                }
+            }
+        }
+    }
+}
+
+module.exports = Frontend;

lib/extension/groups.js

@@ -2,40 +2,60 @@ const settings = require('../util/settings');
 const logger = require('../util/logger');
 const Extension = require('./extension');
 const utils = require('../util/utils');
-const postfixes = utils.getEndpointNames();
+const stringify = require('json-stable-stringify-without-jsonify');
+const equals = require('fast-deep-equal/es6');
 
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
 const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
 
+const stateProperties = {
+    'state': (value, definition) => true,
+    'brightness': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
 class Groups extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.onStateChange = this.onStateChange.bind(this);
         this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-
-            for (let step = 1; step < 20; step++) {
-                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-                this.mqtt.subscribe(`${topic}remove`);
-                this.mqtt.subscribe(`${topic}add`);
-                this.mqtt.subscribe(`${topic}remove_all`);
-            }
-        }
+        this.lastOptimisticState = {};
     }
 
     async onZigbeeStarted() {
+        this.eventBus.on('stateChange', this.onStateChange, this.constructor.name);
         await this.syncGroupsWithSettings();
-        this.eventBus.on('stateChange', this.onStateChange);
     }
 
     async syncGroupsWithSettings() {
         const settingsGroups = settings.getGroups();
         const zigbeeGroups = this.zigbee.getGroups();
+
+        const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
+            try {
+                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
+                if (action === 'remove') {
+                    await endpoint.removeFromGroup(group);
+                } else {
+                    await endpoint.addToGroup(group);
+                }
+            } catch (error) {
+                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
+            }
+        };
+
         for (const settingGroup of settingsGroups) {
             const groupID = settingGroup.ID;
             const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
@@ -48,28 +68,24 @@ class Groups extends Extension {
             // In settings but not in zigbee
             for (const entity of settingsEntity) {
                 if (!zigbeeGroup.hasMember(entity.endpoint)) {
-                    logger.info(`Adding '${entity.name}' to group '${settingGroup.friendlyName}'`);
-                    await entity.endpoint.addToGroup(zigbeeGroup);
+                    addRemoveFromGroup('add', entity.name, settingGroup.friendlyName, entity.endpoint, zigbeeGroup);
                 }
             }
 
             // In zigbee but not in settings
             for (const endpoint of zigbeeGroup.members) {
                 if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
-                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
-                    logger.info(`Removing '${deviceSettings.friendlyName}' from group '${settingGroup.friendlyName}'`);
-                    await endpoint.removeFromGroup(zigbeeGroup);
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, settingGroup.friendlyName, endpoint, zigbeeGroup);
                 }
             }
         }
 
-        // eslint-disable-next-line
         for (const zigbeeGroup of zigbeeGroups) {
             if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
                 for (const endpoint of zigbeeGroup.members) {
-                    const deviceSettings = settings.getDevice(endpoint.getDevice().ieeeAddr);
-                    logger.info(`Removing '${deviceSettings.friendlyName}' from group ${zigbeeGroup.groupID}`);
-                    await endpoint.removeFromGroup(zigbeeGroup);
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
+                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.groupID, endpoint, zigbeeGroup);
                 }
             }
         }
@@ -81,34 +97,61 @@ class Groups extends Extension {
             return;
         }
 
-        const properties = ['state', 'brightness', 'color_temp', 'color'];
         const payload = {};
 
-        properties.forEach((prop) => {
-            if (data.to.hasOwnProperty(prop)) {
-                payload[prop] = data.to[prop];
+        let endpointName;
+        for (let [prop, value] of Object.entries(data.update)) {
+            const endpointNameMatch = utils.endpointNames.find((n) => prop.endsWith(`_${n}`));
+            if (endpointNameMatch) {
+                prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
+                endpointName = endpointNameMatch;
             }
-        });
+
+            if (prop in stateProperties) {
+                payload[prop] = value;
+            }
+        }
 
         if (Object.keys(payload).length) {
-            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
+            const resolvedEntity = this.zigbee.resolveEntity(`${data.ID}${endpointName ? `/${endpointName}` : ''}`);
             const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
                 const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
-                return settingsGroup && settingsGroup.optimistic;
+                return settingsGroup && (!settingsGroup.hasOwnProperty('optimistic') || settingsGroup.optimistic);
             });
 
             if (resolvedEntity.type === 'device') {
                 for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
+                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint) &&
+                        !equals(this.lastOptimisticState[zigbeeGroup.groupID], payload)) {
                         if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                            this.lastOptimisticState[zigbeeGroup.groupID] = payload;
                             await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
                         }
                     }
                 }
             } else {
+                // Invalidate the last optimistic group state when group state is changed directly.
+                delete this.lastOptimisticState[resolvedEntity.group.groupID];
+
                 const groupIDsToPublish = new Set();
                 for (const member of resolvedEntity.group.members) {
-                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
+                    const resolvedEntity = this.zigbee.resolveEntity(member);
+                    const memberPayload = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], resolvedEntity.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = resolvedEntity.endpointName;
+                    if (endpointName) {
+                        Object.keys(memberPayload).forEach((key) => {
+                            memberPayload[`${key}_${endpointName}`] = memberPayload[key];
+                            delete memberPayload[key];
+                        });
+                    }
+
+                    await this.publishEntityState(member.getDevice().ieeeAddr, memberPayload, reason);
                     for (const zigbeeGroup of zigbeeGroups) {
                         if (zigbeeGroup.hasMember(member)) {
                             if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
@@ -143,32 +186,41 @@ class Groups extends Extension {
         let resolvedEntityGroup = null;
         let resolvedEntityDevice = null;
         let hasEndpointName = null;
+        let error = null;
+        let groupKey = null;
+        let deviceKey = null;
+        let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
 
         /* istanbul ignore else */
-        if (this.legacyApi) {
-            const topicMatch = topic.match(legacyTopicRegex);
-            if (topicMatch) {
-                resolvedEntityGroup = this.zigbee.resolveEntity(topicMatch[1]);
-                type = topicMatch[2];
+        const topicRegexMatch = topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
+                type = legacyTopicRegexMatch[2];
 
                 if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
-                    logger.error(`Group '${topicMatch[1]}' does not exist`);
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const payload = {friendly_name: message, group: topicMatch[1], error: 'group doesn\'t exists'};
+                        const payload = {
+                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
+                        };
                         this.mqtt.publish(
                             'bridge/log',
-                            JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
                         );
                     }
 
                     return {};
                 }
-            } else if (topic.match(legacyTopicRegexRemoveAll)) {
-                type = 'remove_all';
             } else {
-                return {};
+                type = 'remove_all';
             }
 
             resolvedEntityDevice = this.zigbee.resolveEntity(message);
@@ -177,89 +229,146 @@ class Groups extends Extension {
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: message, group: topicMatch[1], error: 'entity doesn\'t exists'};
+                    const payload = {
+                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `device_group_${type}_failed`, message: payload}),
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
                     );
                 }
 
                 return {};
             }
 
-            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
+            hasEndpointName = utils.endpointNames.find((p) => message.endsWith(`/${p}`));
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1];
+            message = JSON.parse(message);
+            deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
+                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
+            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
+                error = `Device '${message.device}' does not exist`;
+            }
+
+            hasEndpointName = utils.endpointNames.find((p) => message.device.endsWith(`/${p}`));
         }
 
-        return {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName};
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
+            triggeredViaLegacyApi, skipDisableReporting,
+        };
     }
 
     async onMQTTMessage(topic, message) {
-        const {resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName} =
-            this.parseMQTTMessage(topic, message);
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
+            groupKey, deviceKey, skipDisableReporting,
+        } = this.parseMQTTMessage(topic, message);
         if (!type) return;
+        message = utils.parseJSON(message, message);
 
-        const keys = [
-            `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
-            `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
-        ];
-
-        const definition = resolvedEntityDevice.definition;
-        const endpoints = definition && definition.endpoint ? definition.endpoint(resolvedEntityDevice.device) : null;
-        const endpointName = endpoints ?
-            Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
-
-        if (endpointName) {
-            keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
-            keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+        const responseData = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
         }
 
-        if (!hasEndpointName) {
-            keys.push(resolvedEntityDevice.name);
-            keys.push(resolvedEntityDevice.device.ieeeAddr);
-        }
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
+                ];
 
-        if (type === 'add') {
-            logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
-            settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_add`, message: payload}),
-                );
-            }
-        } else if (type === 'remove') {
-            logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
-            await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
-            settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `device_group_remove`, message: payload}),
-                );
-            }
-        } else { // remove_all
-            logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
-            await resolvedEntityDevice.endpoint.removeFromAllGroups();
-            for (const settingsGroup of settings.getGroups()) {
-                settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const payload = {friendly_name: resolvedEntityDevice.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `device_group_remove_all`, message: payload}),
-                    );
+                const definition = resolvedEntityDevice.definition;
+                const endpoints = definition && definition.endpoint ?
+                    definition.endpoint(resolvedEntityDevice.device) : null;
+                let endpointName = null;
+                if (endpoints) {
+                    const matches = Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID);
+                    endpointName = matches && matches[0];
                 }
+
+                if (endpointName) {
+                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
+                }
+
+                if (!hasEndpointName) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.device.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
+                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
             }
         }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emit('groupMembersChanged',
+                {group: resolvedEntityGroup, action: type, endpoint: resolvedEntityDevice.endpoint,
+                    skipDisableReporting: skipDisableReporting});
+        }
     }
 }
 

lib/extension/legacy/bridgeLegacy.js

@@ -4,6 +4,7 @@ const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 const utils = require('../../util/utils');
 const assert = require('assert');
 const Extension = require('../extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
@@ -52,6 +53,7 @@ class BridgeLegacy extends Extension {
             'device_options': this.deviceOptions,
             'add_group': this.addGroup,
             'remove_group': this.removeGroup,
+            'force_remove_group': this.removeGroup,
             'whitelist': this.whitelist,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
         };
@@ -65,7 +67,7 @@ class BridgeLegacy extends Extension {
             logger.info(`Whitelisted '${entity.friendlyName}'`);
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
             );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
@@ -88,8 +90,8 @@ class BridgeLegacy extends Extension {
 
         const entity = settings.getEntity(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
-        settings.changeDeviceOptions(entity.ID, json.options);
-        logger.info(`Changed device specific options of '${json.friendly_name}' (${JSON.stringify(json.options)})`);
+        settings.changeEntityOptions(entity.ID, json.options);
+        logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
     }
 
     async permitJoin(topic, message) {
@@ -175,9 +177,11 @@ class BridgeLegacy extends Extension {
         });
 
         if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, JSON.stringify(devices), {});
+            this.mqtt.publish(
+                `bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false,
+            );
         } else {
-            this.mqtt.publish('bridge/log', JSON.stringify({type: 'devices', message: devices}));
+            this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
         }
     }
 
@@ -188,7 +192,7 @@ class BridgeLegacy extends Extension {
             return group;
         });
 
-        this.mqtt.publish('bridge/log', JSON.stringify({type: 'groups', message: payload}));
+        this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
     }
 
     rename(topic, message) {
@@ -227,12 +231,13 @@ class BridgeLegacy extends Extension {
             settings.changeFriendlyName(from, to);
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
-            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
+            const eventData = isGroup ? {group: entity.group} : {device: entity.device, from, to};
+            eventData.homeAssisantRename = false;
             this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
 
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
+                stringify({type: `${isGroup ? 'group' : 'device'}_renamed`, message: {from, to}}),
             );
         } catch (error) {
             logger.error(`Failed to rename - ${from} to ${to}`);
@@ -264,7 +269,7 @@ class BridgeLegacy extends Extension {
 
         const group = settings.addGroup(name, id);
         this.zigbee.createGroup(group.ID);
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_added`, message: name}));
+        this.mqtt.publish('bridge/log', stringify({type: `group_added`, message: name}));
         logger.info(`Added group '${name}'`);
     }
 
@@ -272,9 +277,15 @@ class BridgeLegacy extends Extension {
         const name = message;
         const entity = this.zigbee.resolveEntity(message);
         assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+
+        if (topic.includes('force')) {
+            entity.group.removeFromDatabase();
+        } else {
+            entity.group.removeFromNetwork();
+        }
         settings.removeGroup(message);
-        entity.group.removeFromDatabase();
-        this.mqtt.publish('bridge/log', JSON.stringify({type: `group_removed`, message}));
+
+        this.mqtt.publish('bridge/log', stringify({type: `group_removed`, message}));
         logger.info(`Removed group '${name}'`);
     }
 
@@ -301,13 +312,13 @@ class BridgeLegacy extends Extension {
         if (!entity) {
             logger.error(`Cannot ${lookup[action][2]}, device '${message}' does not exist`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
             return;
         }
 
         const cleanup = () => {
             // Fire event
-            this.eventBus.emit('deviceRemoved', {device: entity.device});
+            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
 
             // Remove from configuration.yaml
             settings.removeDevice(entity.settings.ID);
@@ -316,7 +327,7 @@ class BridgeLegacy extends Extension {
             this.state.remove(entity.settings.ID);
 
             logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
@@ -333,7 +344,7 @@ class BridgeLegacy extends Extension {
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_${lookup[action][0]}_failed`, message}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}_failed`, message}));
         }
 
         if (action === 'ban') {
@@ -342,8 +353,6 @@ class BridgeLegacy extends Extension {
     }
 
     async onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
         await this.publish();
     }
 
@@ -364,18 +373,19 @@ class BridgeLegacy extends Extension {
     }
 
     async publish() {
-        const info = await utils.getZigbee2mqttVersion();
+        const info = await utils.getZigbee2MQTTVersion();
         const coordinator = await this.zigbee.getCoordinatorVersion();
         const topic = `bridge/config`;
         const payload = {
             version: info.version,
             commit: info.commitHash,
             coordinator,
+            network: await this.zigbee.getNetworkParameters(),
             log_level: logger.getLevel(),
-            permit_join: await this.zigbee.getPermitJoin(),
+            permit_join: this.zigbee.getPermitJoin(),
         };
 
-        await this.mqtt.publish(topic, JSON.stringify(payload), {retain: true, qos: 0});
+        await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
@@ -386,7 +396,7 @@ class BridgeLegacy extends Extension {
         if (type === 'deviceJoined') {
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
+                stringify({type: `device_connected`, message: {friendly_name: resolvedEntity.name}}),
             );
         } else if (type === 'deviceInterview') {
             if (data.status === 'successful') {
@@ -395,20 +405,20 @@ class BridgeLegacy extends Extension {
                     const log = {friendly_name: resolvedEntity.name, model, vendor, description, supported: true};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta: log}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta: log}),
                     );
                 } else {
                     const meta = {friendly_name: resolvedEntity.name, supported: false};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_successful', meta}),
+                        stringify({type: `pairing`, message: 'interview_successful', meta}),
                     );
                 }
             } else if (data.status === 'failed') {
                 const meta = {friendly_name: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `pairing`, message: 'interview_failed', meta}),
+                    stringify({type: `pairing`, message: 'interview_failed', meta}),
                 );
             } else {
                 /* istanbul ignore else */
@@ -416,13 +426,13 @@ class BridgeLegacy extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `pairing`, message: 'interview_started', meta}),
+                        stringify({type: `pairing`, message: 'interview_started', meta}),
                     );
                 }
             }
         } else if (type === 'deviceAnnounce') {
             const meta = {friendly_name: resolvedEntity.name};
-            this.mqtt.publish('bridge/log', JSON.stringify({type: `device_announced`, message: 'announce', meta}));
+            this.mqtt.publish('bridge/log', stringify({type: `device_announced`, message: 'announce', meta}));
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
@@ -430,7 +440,7 @@ class BridgeLegacy extends Extension {
                 const meta = {friendly_name: name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `device_removed`, message: 'left_network', meta}),
+                    stringify({type: `device_removed`, message: 'left_network', meta}),
                 );
             }
         }
@@ -440,21 +450,21 @@ class BridgeLegacy extends Extension {
         logger.info('Starting touchlink factory reset...');
         this.mqtt.publish(
             'bridge/log',
-            JSON.stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
+            stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
         );
-        const result = await this.zigbee.touchlinkFactoryReset();
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
+                stringify({type: `touchlink`, message: 'reset_success', meta: {status: 'success'}}),
             );
         } else {
             logger.warn('Failed to factory reset device through Touchlink');
             this.mqtt.publish(
                 'bridge/log',
-                JSON.stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
+                stringify({type: `touchlink`, message: 'reset_failed', meta: {status: 'failed'}}),
             );
         }
     }

lib/extension/legacy/deviceGroupMembership.js

@@ -6,13 +6,6 @@ const Extension = require('../extension');
 const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
 
 class DeviceGroupMembership extends Extension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
-            this.mqtt.subscribe(topic);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -29,7 +22,18 @@ class DeviceGroupMembership extends Extension {
             `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
         );
 
-        const {grouplist, capacity} = response;
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendlyName : gid;
+        });
+
         const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
         let msgCapacity;
         if (capacity === 254) {

lib/extension/legacy/report.js

@@ -1,10 +1,9 @@
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const logger = require('../util/logger');
+const logger = require('../../util/logger');
+const settings = require('../../util/settings');
 const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
-const utils = require('../util/utils');
-const Extension = require('./extension');
-const debounce = require('debounce');
-const ZigbeeHerdsman = require('zigbee-herdsman');
+const utils = require('../../util/utils');
+const Extension = require('../extension');
 
 const defaultConfiguration = {
     minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
@@ -15,6 +14,7 @@ const devicesNotSupportingReporting = [
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
     zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0115'),
 ];
 
 const reportKey = 1;
@@ -58,54 +58,12 @@ const clusters = {
     ],
 };
 
-const pollOnMessage = [
-    {
-        // Key is used this.pollDebouncers uniqueness
-        key: 1,
-        // On messages that have the cluster and type of below
-        cluster: {
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 2}},
-                {type: 'commandHueNotification', data: {button: 3}},
-            ],
-            genLevelCtrl: [
-                {type: 'commandStep', data: {}},
-                {type: 'commandStepWithOnOff', data: {}},
-                {type: 'commandStop', data: {}},
-                {type: 'commandMoveWithOnOff', data: {}},
-                {type: 'commandStopWithOnOff', data: {}},
-                {type: 'commandMove', data: {}},
-            ],
-        },
-        // Read the following attributes
-        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerID
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-    },
-    {
-        key: 2,
-        cluster: {
-            genOnOff: [
-                {type: 'commandOn', data: {}},
-                {type: 'commandOff', data: {}},
-                {type: 'commandOffWithEffect', data: {}},
-            ],
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 1}},
-                {type: 'commandHueNotification', data: {button: 4}},
-            ],
-        },
-        read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-    },
-];
-
 class Report extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.queue = new Set();
         this.failed = new Set();
-        this.pollDebouncers = {};
+        this.enabled = settings.get().advanced.report;
     }
 
     shouldIgnoreClusterForDevice(cluster, definition) {
@@ -124,34 +82,48 @@ class Report extends Extension {
         if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
         this.queue.add(device.ieeeAddr);
 
+        const term1 = this.enabled ? 'Setup' : 'Disable';
+        const term2 = this.enabled ? 'setup' : 'disabled';
+
         try {
             for (const ep of device.endpoints) {
                 for (const [cluster, configuration] of Object.entries(clusters)) {
                     if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
-                        logger.debug(`Setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
 
                         const items = [];
                         for (const entry of configuration) {
                             if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
-                                items.push({...entry});
+                                const toAdd = {...entry};
+                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
+                                items.push(toAdd);
                                 delete items[items.length - 1].condition;
                             }
                         }
 
-                        await ep.bind(cluster, this.coordinatorEndpoint);
+                        this.enabled ?
+                            await ep.bind(cluster, this.coordinatorEndpoint) :
+                            await ep.unbind(cluster, this.coordinatorEndpoint);
+
                         await ep.configureReporting(cluster, items);
                         logger.info(
-                            `Successfully setup reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
                         );
                     }
                 }
             }
 
-            // eslint-disable-next-line
-            device.meta.reporting = reportKey;
+            if (this.enabled) {
+                device.meta.reporting = reportKey;
+            } else {
+                delete device.meta.reporting;
+                this.eventBus.emit('reportingDisabled', {device});
+            }
+
+            this.eventBus.emit(`devicesChanged`);
         } catch (error) {
             logger.error(
-                `Failed to setup reporting for '${device.ieeeAddr}' - ${error.stack}`,
+                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
             );
 
             this.failed.add(device.ieeeAddr);
@@ -170,13 +142,33 @@ class Report extends Extension {
         // This message is typically send when a device comes online after being powered off
         // Ikea TRADFRI tend to forget their reporting after powered off.
         // Re-setup reporting.
+        // Only resetup reporting if configuredReportings was not populated yet,
+        // else reconfigure is done in zigbee-herdsman-converters ikea.js/bulbOnEvent
+        // configuredReportings are saved since Zigbee2MQTT 1.17.0
         // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
+        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device) &&
+            device.endpoints.filter((e) => e.configuredReportings.length === 0).length === device.endpoints.length) {
+            return true;
+        }
 
-        if (device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) return false;
-        if (!utils.isRouter(device) || utils.isBatteryPowered(device)) return false;
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
+
+        if (resolvedEntity.device.interviewing === true) return false;
+        if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
         // Gledopto devices don't support reporting.
         if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
+
+        if (this.enabled && device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) {
+            return false;
+        }
+
+        if (!this.enabled && !device.meta.hasOwnProperty('reporting')) {
+            return false;
+        }
+
         return true;
     }
 
@@ -186,69 +178,14 @@ class Report extends Extension {
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
             if (this.shouldSetupReporting(resolvedEntity, null)) {
-                this.setupReporting(resolvedEntity);
+                await this.setupReporting(resolvedEntity);
             }
         }
     }
 
-    onZigbeeEvent(type, data, resolvedEntity) {
+    async onZigbeeEvent(type, data, resolvedEntity) {
         if (this.shouldSetupReporting(resolvedEntity, type)) {
-            this.setupReporting(resolvedEntity);
-        }
-
-        if (type === 'message') {
-            this.poll(data);
-        }
-    }
-
-    poll(message) {
-        /**
-         * This method poll bound endpoints and group members for state changes.
-         *
-         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
-         * Hue bulbs only report their on/off state.
-         * When dimming the bulb via the dimmer switch the state is therefore not reported.
-         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
-         */
-        const polls = pollOnMessage.filter((p) =>
-            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
-            utils.equalsPartial(message.data, c.data)),
-        );
-
-        if (polls.length) {
-            let toPoll = [];
-
-            // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
-                e.binds.map((e) => e).filter((e) => e.target))));
-            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
-            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
-            toPoll = toPoll.map((e) => e.target);
-
-            // If message is published to a group, add members of the group
-            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
-            if (group) {
-                toPoll = toPoll.concat(group.members);
-            }
-
-            toPoll = new Set(toPoll);
-
-            for (const endpoint of toPoll) {
-                for (const poll of polls) {
-                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
-                        continue;
-                    }
-
-                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${poll.key}`;
-                    if (!this.pollDebouncers[key]) {
-                        this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, poll.read.attributes);
-                        }, 1000);
-                    }
-
-                    this.pollDebouncers[key]();
-                }
-            }
+            await this.setupReporting(resolvedEntity);
         }
     }
 }

lib/extension/legacy/softReset.js

@@ -12,11 +12,11 @@ class SoftReset extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
+        this.timeout = utils.seconds(settings.get().advanced.soft_reset_timeout);
     }
 
     onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+        logger.debug(`Soft reset timeout set to ${this.timeout / 1000} seconds`);
         this.resetTimer();
     }
 

lib/extension/networkMap.js

@@ -2,6 +2,7 @@ const settings = require('../util/settings');
 const utils = require('../util/utils');
 const logger = require('../util/logger');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 /**
  * This extension creates a network map
@@ -13,26 +14,21 @@ class NetworkMap extends Extension {
         this.legacyApi = settings.get().advanced.legacy_api;
         this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
         this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
 
         // Bind
         this.raw = this.raw.bind(this);
         this.graphviz = this.graphviz.bind(this);
+        this.plantuml = this.plantuml.bind(this);
 
         // Set supported formats
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
+            'plantuml': this.plantuml,
         };
     }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-            this.mqtt.subscribe(this.legacyTopicRoutes);
-        }
-    }
-
     async onMQTTMessage(topic, message) {
         /* istanbul ignore else */
         if (this.legacyApi) {
@@ -40,14 +36,38 @@ class NetworkMap extends Extension {
                 this.supportedFormats.hasOwnProperty(message)) {
                 const includeRoutes = topic === this.legacyTopicRoutes;
                 const topology = await this.networkScan(includeRoutes);
-                const converted = this.supportedFormats[message](topology);
+                let converted = this.supportedFormats[message](topology);
+                converted = message === 'raw' ? stringify(converted) : converted;
                 this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
             }
         }
+
+        if (topic === this.topic) {
+            try {
+                message = utils.parseJSON(message, message);
+                const type = typeof message === 'object' ? message.type : message;
+                if (!this.supportedFormats.hasOwnProperty(type)) {
+                    throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
+                }
+
+                const routes = typeof message === 'object' && message.routes;
+                const topology = await this.networkScan(routes);
+                const value = this.supportedFormats[type](topology);
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {routes, type, value}, null)),
+                );
+            } catch (error) {
+                await this.mqtt.publish(
+                    'bridge/response/networkmap',
+                    stringify(utils.getResponse(message, {}, error.message)),
+                );
+            }
+        }
     }
 
     raw(topology) {
-        return JSON.stringify(topology);
+        return topology;
     }
 
     graphviz(topology) {
@@ -64,15 +84,14 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
             if (node.type !== 'Coordinator') {
-                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
-                if (definition) {
-                    labels.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                if (node.definition) {
+                    labels.push(`${node.definition.vendor} ${node.definition.description} (${node.definition.model})`);
                 } else {
                     // This model is not supported by zigbee-herdsman-converters, add zigbee model information
                     labels.push(`${node.manufacturerName} ${node.modelID}`);
@@ -83,7 +102,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'ISO_8601_local');
+                lastSeen = utils.formatDate(date, 'relative');
             }
 
             labels.push(lastSeen);
@@ -113,7 +132,7 @@ class NetworkMap extends Extension {
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
-                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
                 text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
@@ -126,6 +145,64 @@ class NetworkMap extends Extension {
         return text.replace(/\0/g, '');
     }
 
+    plantuml(topology) {
+        const text = [];
+
+        text.push(`' paste into: https://www.planttext.com/`);
+        text.push(``);
+        text.push('@startuml');
+
+        topology.nodes.sort((a, b) => a.friendlyName.localeCompare(b.friendlyName)).forEach((node) => {
+            // Add friendly name
+            text.push(`card ${node.ieeeAddr} [`);
+            text.push(`${node.friendlyName}`);
+            text.push(`---`);
+
+            // Add the device short network address, ieeaddr and scan note (if any)
+            text.push(
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
+                ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
+            );
+
+            // Add the device model
+            if (node.type !== 'Coordinator') {
+                text.push(`---`);
+                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                if (definition) {
+                    text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
+                } else {
+                    // This model is not supported by zigbee-herdsman-converters, add zigbee model information
+                    text.push(`${node.manufacturerName} ${node.modelID}`);
+                }
+            }
+
+            // Add the device last_seen timestamp
+            let lastSeen = 'unknown';
+            const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
+            if (date) {
+                lastSeen = utils.formatDate(date, 'relative');
+            }
+            text.push(`---`);
+            text.push(lastSeen);
+            text.push(`]`);
+            text.push(``);
+        });
+
+        /**
+         * Add edges between the devices
+         * NOTE: There are situations where a device is NOT in the topology, this can be e.g.
+         * due to not responded to the lqi scan. In that case we do not add an edge for this device.
+         */
+        topology.links.forEach((link) => {
+            text.push(`${link.sourceIeeeAddr} --> ${link.targetIeeeAddr}: ${link.lqi}`);
+        });
+        text.push('');
+
+        text.push(`@enduml`);
+
+        return text.join(`\n`);
+    }
+
     async networkScan(includeRoutes) {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
         const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
@@ -135,9 +212,25 @@ class NetworkMap extends Extension {
 
         for (const device of devices.filter((d) => d.type != 'EndDevice')) {
             failed.set(device, []);
+            await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
             const resolvedEntity = this.zigbee.resolveEntity(device);
+
+            const doRequest = async (request, firstAttempt = true) => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return await doRequest(request, false);
+                    }
+                }
+            };
+
             try {
-                const result = await device.lqi();
+                const result = await doRequest(async () => await device.lqi());
                 lqis.set(device, result);
                 logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
             } catch (error) {
@@ -147,7 +240,7 @@ class NetworkMap extends Extension {
 
             if (includeRoutes) {
                 try {
-                    const result = await device.routingTable();
+                    const result = await doRequest(async () => await device.routingTable());
                     routingTables.set(device, result);
                     logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
                 } catch (error) {
@@ -163,10 +256,22 @@ class NetworkMap extends Extension {
         // Add nodes
         for (const device of devices) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
+            const definition = resolvedEntity.definition ? {
+                model: resolvedEntity.definition.model,
+                vendor: resolvedEntity.definition.vendor,
+                description: resolvedEntity.definition.description,
+                supports: Array.from(new Set((resolvedEntity.definition.exposes).map((e) => {
+                    return e.hasOwnProperty('name') ?
+                        e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
+            } : null;
+
             networkMap.nodes.push({
                 ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
                 networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
                 modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+                definition,
             });
         }
 
@@ -178,6 +283,13 @@ class NetworkMap extends Extension {
                     continue;
                 }
 
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.getDeviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
                 const link = {
                     source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
                     target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},

lib/extension/onEvent.js

@@ -19,6 +19,7 @@ class OnEvent extends Extension {
     }
 
     async stop() {
+        super.stop();
         for (const device of this.zigbee.getClients()) {
             const resolvedEntity = this.zigbee.resolveEntity(device);
             this.callOnEvent(resolvedEntity, 'stop', {});
@@ -26,10 +27,10 @@ class OnEvent extends Extension {
     }
 
     callOnEvent(resolvedEntity, type, data) {
-        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device);
+        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
 
         if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
-            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device);
+            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
         }
     }
 }

lib/extension/otaUpdate.js

@@ -1,9 +1,13 @@
 const settings = require('../util/settings');
 const logger = require('../util/logger');
-const assert = require('assert');
+const stringify = require('json-stable-stringify-without-jsonify');
+const utils = require('../util/utils');
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
+const tradfriOTA = require('zigbee-herdsman-converters/lib/ota/tradfri');
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
+
 const Extension = require('./extension');
-const MINUTES_10 = 1000 * 60 * 10;
 
 class OTAUpdate extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
@@ -11,31 +15,44 @@ class OTAUpdate extends Extension {
         this.inProgress = new Set();
         this.lastChecked = {};
         this.legacyApi = settings.get().advanced.legacy_api;
-    }
 
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+        if (settings.get().advanced.ikea_ota_use_test_url) {
+            tradfriOTA.useTestURL();
         }
     }
 
+    onMQTTConnected() {
+        for (const device of this.zigbee.getClients()) {
+            // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
+            // remove them.
+            this.removeProgressAndRemainingFromState(device);
+        }
+    }
+
+    removeProgressAndRemainingFromState(device) {
+        this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
+        this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
+    }
+
     async onZigbeeEvent(type, data, resolvedEntity) {
+        if (settings.get().ota.disable_automatic_update_check) return;
         if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+        logger.debug(`Device '${resolvedEntity.name}' requested OTA`);
 
         const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
-            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
-            // each time.
+            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
+            // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
+            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
             const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
-                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
             const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
-            this.publishEntityState(data.device.ieeeAddr, {update_available: available});
+            const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+            this.publishEntityState(data.device.ieeeAddr, payload);
 
             if (available) {
                 const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
@@ -46,7 +63,7 @@ class OTAUpdate extends Extension {
                     const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        stringify({type: `ota_update`, message, meta}),
                     );
                 }
             }
@@ -79,141 +96,177 @@ class OTAUpdate extends Extension {
         }
     }
 
-    async onMQTTMessage(topic, message) {
-        let resolvedEntity = null;
+    getEntityPublishPayload(state, progress=null, remaining=null) {
+        const payload = {update: {state}};
+        if (progress !== null) payload.update.progress = progress;
+        if (remaining !== null) payload.update.remaining = Math.round(remaining);
+
         /* istanbul ignore else */
         if (this.legacyApi) {
-            if (!topic.match(legacyTopicRegex)) {
-                return null;
-            }
+            payload.update_available = state === 'available';
+        }
 
-            resolvedEntity = this.zigbee.resolveEntity(message);
-        } else {
+        return payload;
+    }
+
+    async onMQTTMessage(topic, message) {
+        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
             return null;
         }
 
-        assert(resolvedEntity != null && resolvedEntity.type === 'device', 'Device not found or not a device');
-        if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
-            const message = `Device '${resolvedEntity.name}' does not support OTA updates`;
-            logger.error(message);
+        message = utils.parseJSON(message, message);
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+        const resolvedEntity = this.zigbee.resolveEntity(ID);
+        const type = topic.substring(topic.lastIndexOf('/') + 1);
+        const responseData = {id: ID};
+        let error = null;
+        let errorStack = null;
+
+        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+            error = `Device '${ID}' does not exist`;
+        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
+            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
                 const meta = {status: `not_supported`, device: resolvedEntity.name};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
+                    stringify({type: `ota_update`, message: error, meta}),
                 );
             }
+        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else {
+            this.inProgress.add(resolvedEntity.device.ieeeAddr);
 
-            return;
-        }
-
-        if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
-            logger.error(`Update or check already in progress for '${resolvedEntity.name}', skipping...`);
-            return;
-        }
-        this.inProgress.add(resolvedEntity.device.ieeeAddr);
-
-        const type = topic.substring(settings.get().mqtt.base_topic.length).split('/')[3];
-        if (type === 'check') {
-            const message = `Checking if update available for '${resolvedEntity.name}'`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `checking_if_available`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const available = await resolvedEntity.definition.ota.isUpdateAvailable(resolvedEntity.device, logger);
-                const message = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
-                logger.info(message);
+            if (type === 'check') {
+                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: available});
-                this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
-            } catch (error) {
-                const message = `Failed to check if update available for '${resolvedEntity.name}' (${error.message})`;
-                logger.error(message);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `check_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        JSON.stringify({type: `ota_update`, message, meta}),
+                try {
+                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
+                        resolvedEntity.device, logger,
                     );
-                }
-            }
-        } else { // type === 'update'
-            const message = `Updating '${resolvedEntity.name}' to latest firmware`;
-            logger.info(message);
-
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const meta = {status: `update_in_progress`, device: resolvedEntity.name};
-                this.mqtt.publish(
-                    'bridge/log',
-                    JSON.stringify({type: `ota_update`, message, meta}),
-                );
-            }
-
-            try {
-                const onProgress = (progress, remaining) => {
-                    let message = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
-                    if (remaining) {
-                        message += `, +- ${Math.round(remaining / 60)} minutes remaining`;
-                    }
-
-                    logger.info(message);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
-                        this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: msg, meta}),
+                        );
                     }
-                };
 
-                const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
-                await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
-                const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
-                const [fromS, toS] = [JSON.stringify(from_), JSON.stringify(to)];
-                const message = `Finished update of '${resolvedEntity.name}'` +
-                    (to ? `, from '${fromS}' to '${toS}'` : ``);
-                logger.info(message);
-                this.publishEntityState(resolvedEntity.device.ieeeAddr, {update_available: false});
+                    const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    responseData.updateAvailable = available;
+                } catch (e) {
+                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+                    errorStack = e.stack;
 
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `ota_update`, message: error, meta}),
+                        );
+                    }
                 }
-            } catch (error) {
-                const message = `Update of '${resolvedEntity.name}' failed (${error.message})`;
-                logger.error(message);
+            } else { // type === 'update'
+                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_failed`, device: resolvedEntity.name};
-                    this.mqtt.publish('bridge/log', JSON.stringify({type: `ota_update`, message, meta}));
+                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `ota_update`, message: msg, meta}),
+                    );
+                }
+
+                try {
+                    const onProgress = (progress, remaining) => {
+                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                        if (remaining) {
+                            msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
+                        }
+
+                        logger.info(msg);
+
+                        const payload = this.getEntityPublishPayload('updating', progress, remaining);
+                        this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
+                        }
+                    };
+
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
+                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const [fromS, toS] = [stringify(from_), stringify(to)];
+                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                        (to ? `, from '${fromS}' to '${toS}'` : ``);
+                    logger.info(msg);
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('idle');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    responseData.from = from_ ? utils.toSnakeCase(from_) : null;
+                    responseData.to = to ? utils.toSnakeCase(to) : null;
+                    this.eventBus.emit(`devicesChanged`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
+                    }
+                } catch (e) {
+                    logger.debug(`Update of '${resolvedEntity.name}' failed (${e})`);
+                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+                    errorStack = e.stack;
+
+                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    const payload = this.getEntityPublishPayload('available');
+                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
+                    }
                 }
             }
+
+            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
         }
 
-        this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+            errorStack && logger.debug(errorStack);
+        }
     }
 }
 

lib/extension/publish.js

@@ -5,35 +5,28 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const assert = require('assert');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
-const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}))?/(get|set)(?:/(.+))?`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
 
-const groupConverters = [
+const defaultGroupConverters = [
     zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_color,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
+    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
     zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
     zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
     zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
     zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
     zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
 ];
 
 class EntityPublish extends Extension {
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < 20; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
     parseTopic(topic) {
         const match = topic.match(topicRegex);
         if (!match) {
@@ -64,7 +57,7 @@ class EntityPublish extends Extension {
                 const message = {friendly_name: entityKey};
                 this.mqtt.publish(
                     'bridge/log',
-                    JSON.stringify({type: `entity_not_found`, message}),
+                    stringify({type: `entity_not_found`, message}),
                 );
             }
 
@@ -78,6 +71,7 @@ class EntityPublish extends Extension {
         let options = {};
         let device = null;
         let definition = null;
+        let membersState = null;
 
         assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
         if (resolvedEntity.type === 'device') {
@@ -93,37 +87,56 @@ class EntityPublish extends Extension {
             converters = resolvedEntity.definition.toZigbee;
             options = resolvedEntity.settings;
         } else {
-            converters = groupConverters;
             target = resolvedEntity.group;
             options = resolvedEntity.settings;
-            definition = resolvedEntity.group.members.map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice()));
+            definition = resolvedEntity.group.members
+                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
+            converters = new Set();
+            definition.forEach((d) => d.toZigbee.forEach(converters.add, converters));
+            converters = converters.size ? [...converters] : defaultGroupConverters;
+            membersState = {};
+            for (const member of resolvedEntity.group.members) {
+                const ieeeAddr = member.getDevice().ieeeAddr;
+                membersState[ieeeAddr] = this.state.get(ieeeAddr);
+            }
         }
 
         // Convert the MQTT message to a Zigbee message.
         let json = {};
         if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            json[topic.attribute] = message;
+            try {
+                json[topic.attribute] = JSON.parse(message);
+            } catch (e) {
+                json[topic.attribute] = message;
+            }
         } else {
             try {
                 json = JSON.parse(message);
             } catch (e) {
-                // Cannot be parsed to JSON, assume state message.
-                json = {state: message};
+                if (stateValues.includes(message.toLowerCase())) {
+                    json = {state: message};
+                } else {
+                    logger.error(`Invalid JSON '${message}', skipping...`);
+                    return false;
+                }
             }
         }
-
+        if (!json) {
+            logger.error(`Invalid JSON '${message}', skipping...`);
+            return;
+        }
 
         /**
          * Home Assistant always publishes 'state', even when e.g. only setting
          * the color temperature. This would lead to 2 zigbee publishes, where the first one
          * (state) is probably unecessary.
          */
-        const deviceState = this.state.get(resolvedEntity.settings.ID) || {};
+        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
         if (settings.get().homeassistant) {
             const hasColorTemp = json.hasOwnProperty('color_temp');
             const hasColor = json.hasOwnProperty('color');
             const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = deviceState && deviceState.state === 'ON' ? true : false;
+            const isOn = entityState && entityState.state === 'ON' ? true : false;
             if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
                 delete json.state;
                 logger.debug('Skipping state because of Home Assistant');
@@ -145,20 +158,28 @@ class EntityPublish extends Extension {
 
         // For each attribute call the corresponding converter
         const usedConverters = {};
+        const toPublish = {};
+        const addToToPublish = (ID, payload) => {
+            if (!(ID in toPublish)) toPublish[ID] = {};
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
         for (let [key, value] of entries) {
             let endpointName = topic.endpointName;
             let actualTarget = target;
+            let endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
 
             // When the key has a endpointName included (e.g. state_right), this will override the target.
             if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const underscoreIndex = key.lastIndexOf('_');
-                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possibleEndpointName)) {
-                    endpointName = possibleEndpointName;
-                    key = key.substring(0, underscoreIndex);
+                const endpointNameMatch = utils.endpointNames.find((n) => key.endsWith(`_${n}`));
+                if (endpointNameMatch) {
+                    endpointName = endpointNameMatch;
+                    const regex = new RegExp(`(_${endpointNameMatch})$`, 'gm');
+                    key = key.replace(regex, ``);
+
                     const device = target.getDevice();
                     actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-
+                    endpointOrGroupID = endpointName;
                     if (!actualTarget) {
                         logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
                         continue;
@@ -166,12 +187,12 @@ class EntityPublish extends Extension {
                 }
             }
 
-            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
             if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
             const converter = converters.find((c) => c.key.includes(key));
 
-            if (usedConverters[endpointOrGroupID].includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
+            if (topic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
                 continue;
             }
 
@@ -184,30 +205,53 @@ class EntityPublish extends Extension {
             const meta = {
                 endpoint_name: endpointName,
                 options,
-                message: json,
+                message: {...json},
                 logger,
                 device,
-                state: deviceState,
+                state: entityState,
+                membersState,
                 mapped: definition,
             };
 
+            // Strip endpoint name from meta.message properties.
+            if (meta.endpoint_name) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(meta.endpoint_name)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - meta.endpoint_name.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
             try {
                 if (topic.type === 'set' && converter.convertSet) {
                     logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
                     const result = await converter.convertSet(actualTarget, key, value, meta);
-                    if (result && result.state) {
+                    const optimistic = !options.hasOwnProperty('optimistic') || options.optimistic;
+                    if (result && result.state && optimistic) {
                         const msg = result.state;
 
                         if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
-                                if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${endpointName}`] = msg[key];
-                                    delete msg[key];
-                                }
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
                             }
                         }
 
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
+                        // filter out attribute listed in filtered_optimistic
+                        if (options.hasOwnProperty('filtered_optimistic')) {
+                            options.filtered_optimistic.forEach((a) => delete msg[a]);
+                        }
+                        if (Object.keys(msg).length != 0) {
+                            addToToPublish(resolvedEntity.settings.ID, msg);
+                        }
+                    }
+
+                    if (result && result.membersState && optimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            addToToPublish(ieeeAddr, state);
+                        }
                     }
 
                     // It's possible for devices to get out of sync when writing an attribute that's not reportable.
@@ -217,6 +261,7 @@ class EntityPublish extends Extension {
                     // When reporting is requested for a device (report: true in device-specific settings) we won't
                     // ever issue a read here, as we assume the device will properly report changes.
                     // Only do this when the retrieve_state option is enabled for this device.
+                    // retrieve_state == decprecated
                     if (
                         resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
                         resolvedEntity.settings.retrieve_state
@@ -241,7 +286,7 @@ class EntityPublish extends Extension {
                     const meta = {friendly_name: resolvedEntity.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        JSON.stringify({type: `zigbee_publish_error`, message, meta}),
+                        stringify({type: `zigbee_publish_error`, message, meta}),
                     );
                 }
             }
@@ -249,6 +294,10 @@ class EntityPublish extends Extension {
             usedConverters[endpointOrGroupID].push(converter);
         }
 
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            this.publishEntityState(ID, payload);
+        }
+
         return true;
     }
 }

lib/extension/receive.js

@@ -3,13 +3,14 @@ const logger = require('../util/logger');
 const utils = require('../util/utils');
 const debounce = require('debounce');
 const Extension = require('./extension');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 class Receive extends Extension {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         super(zigbee, mqtt, state, publishEntityState, eventBus);
         this.elapsed = {};
         this.debouncers = {};
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
+        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data), this.constructor.name);
     }
 
     async onZigbeeStarted() {
@@ -73,38 +74,16 @@ class Receive extends Extension {
             return false;
         }
 
-        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
-            logger.debug('Ignoring message from coordinator');
-            return false;
-        }
-
-        /**
-         * Don't handle re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The source address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the source address of the end device.
-         * As the same message is also received directly from the end device, it makes no sense
-         * to handle these messages.
-         */
-        const hasGroupID = data.hasOwnProperty('groupID') && !!data.groupID;
-        if (utils.isXiaomiDevice(data.device) && utils.isRouter(data.device) && hasGroupID) {
-            logger.debug('Skipping re-transmitted Xiaomi message');
-            return false;
-        }
-
-        if (!data.device.modelID && data.device.interviewing) {
-            logger.debug(`Skipping message, modelID is undefined and still interviewing`);
-            return false;
-        }
-
         if (!resolvedEntity.definition) {
-            logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
-            logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            if (data.device.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.modelID}' ` +
+                    `and manufacturer name '${data.device.manufacturerName}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
             return false;
         }
 
@@ -112,6 +91,24 @@ class Receive extends Extension {
     }
 
     onZigbeeEvent(type, data, resolvedEntity) {
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
+            resolvedEntity = this.zigbee.resolveEntity(data.device);
+        }
+
         if (!this.shouldProcess(type, data, resolvedEntity)) {
             return;
         }
@@ -122,28 +119,20 @@ class Receive extends Extension {
         });
 
         // Check if there is an available converter, genOta messages are not interesting.
-        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
+        if (!converters.length && !['genOta', 'genTime', 'genBasic'].includes(data.cluster)) {
             logger.debug(
                 `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
-                `and type '${data.type}' and data '${JSON.stringify(data.data)}'`,
+                `and type '${data.type}' and data '${stringify(data.data)}'`,
             );
             return;
         }
 
-        const debounce = resolvedEntity.settings.debounce || settings.get().device_options.debounce;
-        const debounceIgnore = resolvedEntity.settings.debounce_ignore || settings.get().device_options.debounce_ignore;
-
         // Convert this Zigbee message to a MQTT message.
         // Get payload for the message.
         // - If a payload is returned publish it to the MQTT broker
         // - If NO payload is returned do nothing. This is for non-standard behaviour
         //   for e.g. click switches where we need to count number of clicks and detect long presses.
         const publish = (payload) => {
-            // Add device linkquality.
-            if (data.hasOwnProperty('linkquality')) {
-                payload.linkquality = data.linkquality;
-            }
-
             if (settings.get().advanced.elapsed) {
                 const now = Date.now();
                 if (this.elapsed[data.device.ieeeAddr]) {
@@ -154,18 +143,22 @@ class Receive extends Extension {
             }
 
             // Check if we have to debounce
-            if (debounce) {
-                this.publishDebounce(data.device.ieeeAddr, payload, debounce, debounceIgnore);
+            if (resolvedEntity.settings.debounce) {
+                this.publishDebounce(
+                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
+                    resolvedEntity.settings.debounce_ignore,
+                );
             } else {
                 this.publishEntityState(data.device.ieeeAddr, payload);
             }
         };
 
-        const meta = {device: data.device};
+        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
         let payload = {};
         converters.forEach((converter) => {
-            const options = {...settings.get().device_options, ...settings.getDevice(data.device.ieeeAddr)};
-            const converted = converter.convert(resolvedEntity.definition, data, publish, options, meta);
+            const converted = converter.convert(
+                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
+            );
             if (converted) {
                 payload = {...payload, ...converted};
             }
@@ -173,6 +166,7 @@ class Receive extends Extension {
 
         if (Object.keys(payload).length) {
             publish(payload);
+            return true;
         }
     }
 }

lib/mqtt.js

@@ -8,6 +8,7 @@ class MQTT extends events.EventEmitter {
     constructor() {
         super();
         this.onMessage = this.onMessage.bind(this);
+        this.publishedTopics = new Set();
     }
 
     async connect() {
@@ -18,7 +19,7 @@ class MQTT extends events.EventEmitter {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
-                retain: true,
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
             },
         };
 
@@ -44,8 +45,11 @@ class MQTT extends events.EventEmitter {
         }
 
         if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
         }
 
         if (mqttSettings.client_id) {
@@ -69,9 +73,11 @@ class MQTT extends events.EventEmitter {
         return new Promise((resolve) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
+            const self = this;
             this.client.on('connect', () => {
                 logger.info('Connected to MQTT server');
-                this.publish('bridge/state', 'online', {retain: true, qos: 0});
+                self.subscribe(`${settings.get().mqtt.base_topic}/#`);
+                self.publish('bridge/state', 'online', {retain: true, qos: 0});
                 resolve();
             });
 
@@ -92,27 +98,47 @@ class MQTT extends events.EventEmitter {
     }
 
     onMessage(topic, message) {
-        this.emit('message', {topic, message: message.toString()});
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            this.emit('message', {topic, message: message + ''});
+        }
     }
 
     isConnected() {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic) {
+    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true) {
         topic = `${base}/${topic}`;
         options = {qos: 0, retain: false, ...options};
 
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
+        this.emit('publishMessage', {topic, payload, options});
+
         if (!this.isConnected()) {
-            logger.error(`Not connected to MQTT server!`);
-            logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            if (!skipLog) {
+                logger.error(`Not connected to MQTT server!`);
+                logger.error(`Cannot send message: topic: '${topic}', payload: '${payload}`);
+            }
             return;
         }
 
-        logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        if (!skipLog) {
+            logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
+        }
+
+        const actualOptions = {...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
 
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, options, () => resolve());
+            this.client.publish(topic, payload, actualOptions, () => {
+                resolve();
+            });
         });
     }
 }

lib/state.js

@@ -1,14 +1,15 @@
 const logger = require('./util/logger');
 const data = require('./util/data');
+const settings = require('./util/settings');
 const fs = require('fs');
 const objectAssignDeep = require('object-assign-deep');
 
 const saveInterval = 1000 * 60 * 5; // 5 minutes
 
 const dontCacheProperties = [
-    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
 ];
 
 class State {
@@ -53,9 +54,17 @@ class State {
     }
 
     save() {
-        logger.debug(`Saving state to file ${this.file}`);
-        const json = JSON.stringify(this.state, null, 4);
-        fs.writeFileSync(this.file, json, 'utf8');
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
+        } else {
+            logger.debug(`Not saving state`);
+        }
     }
 
     exists(ID) {
@@ -66,8 +75,10 @@ class State {
         return this.state[ID];
     }
 
-    set(ID, state, reason=null) {
-        const toState = objectAssignDeep.noMutate(state);
+    set(ID, update, reason=null) {
+        const fromState = this.state[ID] || {};
+        const toState = objectAssignDeep.noMutate(fromState, update);
+        const result = {...toState};
 
         for (const property of Object.keys(toState)) {
             if (dontCacheProperties.find((p) => property.match(p))) {
@@ -75,11 +86,27 @@ class State {
             }
         }
 
-        const fromState = this.state[ID];
-
         this.state[ID] = toState;
+        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason, update});
+        return result;
+    }
 
-        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
+    removeKey(ID, path) {
+        if (this.exists(ID)) {
+            let state = this.state[ID];
+            for (let i = 0; i < path.length; i++) {
+                const key = path[i];
+                if (i === path.length - 1) {
+                    delete state[key];
+                } else {
+                    if (state[key]) {
+                        state = state[key];
+                    } else {
+                        break;
+                    }
+                }
+            }
+        }
     }
 
     remove(ID) {

lib/types/types.ts

@@ -0,0 +1,169 @@
+import type {Device as ZHDevice, Endpoint} from 'zigbee-herdsman/dist/controller/model';
+
+declare global {
+    type Device = ZHDevice;
+
+    type ZigbeeEventType = 'deviceLeave' | 'deviceAnnounce';
+
+    interface ZigbeeEventData {
+        ieeeAddr: string;
+    }
+
+    /* eslint-disable */
+    // Controller
+    interface KeyValue {
+        [s: string]: any,
+    }
+
+    interface Settings {
+        devices?: {[s: string]: {friendly_name: string, retention?: number}},
+        groups?: {[s: string]: {friendly_name: string, devices?: string[]}},
+        passlist: string[],
+        blocklist: string[],
+        whitelist: string[],
+        ban: string[],
+        availability?: boolean | {
+            active?: {timeout?: number},
+            passive?: {timeout?: number}
+        },
+        permit_join: boolean,
+        frontend?: {
+            auth_token?: string,
+        },
+        mqtt: {
+            include_device_information: boolean,
+            force_disable_retain: boolean
+            version?: number,
+            user?: string,
+            password?: string,
+        },
+        serial: {
+            disable_led: boolean,
+        },
+        device_options: {[s: string]: unknown},
+        map_options: {
+            graphviz: {
+                colors: {
+                    fill: {
+                        enddevice: string,
+                        coordinator: string,
+                        router: string,
+                    },
+                    font: {
+                        coordinator: string,
+                        router: string,
+                        enddevice: string,
+                    },
+                    line: {
+                        active: string,
+                        inactive: string,
+                    },
+                },
+            },
+        },
+        experimental: {
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            availability_new?: boolean,
+        },
+        advanced: {
+            legacy_api: boolean,
+            log_rotation: boolean,
+            log_symlink_current: boolean,
+            log_output: ('console' | 'file')[],
+            log_directory: string,
+            log_file: string,
+            log_level: 'debug' | 'info' | 'error' | 'warn',
+            log_syslog: {},
+            soft_reset_timeout: number,
+            pan_id: number,
+            ext_pan_id: number[],
+            channel: number,
+            adapter_concurrent: number | null,
+            adapter_delay: number | null,
+            availability_timeout: number,
+            availability_blocklist: string[],
+            availability_passlist: string[],
+            availability_blacklist: string[],
+            availability_whitelist: string[],
+            cache_state: boolean,
+            cache_state_persistent: boolean,
+            cache_state_send_on_startup: boolean,
+            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' |  'epoch',
+            elapsed: boolean,
+            network_key: number[],
+            report: boolean,
+            homeassistant_discovery_topic: string,
+            homeassistant_status_topic: string,
+            homeassistant_legacy_entity_attributes: boolean,
+            homeassistant_legacy_triggers: boolean,
+            timestamp_format: string,
+        },
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean
+        },
+        external_converters: string[],
+    }
+
+    interface DeviceSettings {
+        friendlyName: string,
+        ID: string,
+        retention?: number,
+    }
+
+    interface GroupSettings {
+        friendlyName: string,
+        devices: string[],
+        ID: number,
+    }
+
+    type EntitySettings = {
+        type: 'device' | 'group'
+        ID: number | string,
+        friendlyName: string,
+    }
+
+    interface ResolvedEntity {
+        type: 'device' | 'group',
+    }
+
+    interface Definition  {
+        model: string
+        toZigbee: {key: string[], convertGet?: (entity: Endpoint, key: string, meta: {message: {}, mapped: Definition}) => Promise<void>}[]
+    }
+
+    interface ResolvedDevice {
+        type: 'device',
+        definition?: Definition,
+        name: string,
+        endpoint: Endpoint,
+        device: Device,
+        settings: {
+            friendlyName: string,
+            availability?: {timeout?: number} | boolean,
+        }
+    }
+
+    type lastSeenChangedHandler = (data: {device: Device}) => void;
+
+    interface TempZigbee {
+        getClients: () => Device[];
+        on: (event: 'lastSeenChanged', handler: lastSeenChangedHandler) => void;
+        removeListener: (event: 'lastSeenChanged', handler: lastSeenChangedHandler) => void;
+        resolveEntity: (device: Device) => ResolvedEntity;
+    }
+
+    interface TempMQTT {
+        publish: (topic: string, payload: string, options: {}, base?: string, skipLog?: boolean, skipReceive?: boolean) => Promise<void>;
+    }
+
+    interface TempState {
+        get: (ID: string) => {} | null;
+    }
+
+    interface TempEventBus {
+        removeListenersExtension: (extension: string) => void;
+    }
+
+    type TempPublishEntityState = () => void;
+}

lib/util/logger.js

@@ -19,12 +19,25 @@ const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', time
 // Make sure that log directoy exsists when not logging to stdout only
 if (output.includes('file')) {
     fx.mkdirSync(directory);
+
+    if (settings.get().advanced.log_symlink_current) {
+        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+        const actual = './' + timestamp;
+        if (fs.existsSync(current)) {
+            fs.unlinkSync(current);
+        }
+        fs.symlinkSync(actual, current);
+    }
 }
 
+const z2mToWinstonLevel = (level) => level === 'warn' ? 'warning' : level;
+const winstonToZ2mLevel = (level) => level === 'warning' ? 'warn' : level;
+
 // Determine the log level.
-const level = settings.get().advanced.log_level;
+let level = settings.get().advanced.log_level;
 const validLevels = ['info', 'error', 'warn', 'debug'];
 assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
+level = z2mToWinstonLevel(level);
 
 const levelWithCompensatedLength = {
     'info': 'info ',
@@ -45,8 +58,8 @@ const transportsToUse = [
             winston.format.timestamp({format: timestampFormat}),
             winston.format.printf(/* istanbul ignore next */(info) => {
                 let {timestamp, level, message} = info;
-                level = level === 'warning' ? 'warn' : level;
-                const prefix = colorizer.colorize(level, `zigbee2mqtt:${levelWithCompensatedLength[level]}`);
+                level = winstonToZ2mLevel(level);
+                const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
                 return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
             }),
         ),
@@ -63,7 +76,7 @@ const transportFileOptions = {
         winston.format.timestamp({format: timestampFormat}),
         winston.format.printf(/* istanbul ignore next */(info) => {
             let {timestamp, level, message} = info;
-            level = level === 'warning' ? 'warn' : level;
+            level = winstonToZ2mLevel(level);
             return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
         }),
     ),
@@ -82,7 +95,14 @@ if (output.includes('file')) {
 /* istanbul ignore next */
 if (output.includes('syslog')) {
     require('winston-syslog').Syslog;
-    const options = {app_name: 'zigbee2mqtt', ...settings.get().advanced.log_syslog};
+    const options = {
+        app_name: 'Zigbee2MQTT',
+        format: winston.format.printf(/* istanbul ignore next */(info) => {
+            return `${info.message}`;
+        }),
+        ...settings.get().advanced.log_syslog,
+    };
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
     transportsToUse.push(new winston.transports.Syslog(options));
 }
 
@@ -108,24 +128,32 @@ function cleanup() {
     }
 }
 
-logger.cleanup = cleanup;
-logger.getLevel = () => transportsToUse[0].level;
-logger.setLevel = (level) => {
-    transportsToUse.forEach((transport) => transport.level = level);
-};
-
 // Print to user what logging is enabled
-if (output.includes('file')) {
-    if (output.includes('console')) {
-        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
-    } else {
-        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+function logOutput() {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        logger.cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
     }
-    logger.cleanup();
-} else if (output.includes('console')) {
-    logger.info(`Logging to console only'`);
 }
 
+logger.addTransport = (transport) => {
+    transport.level = transportsToUse[0].level;
+    logger.add(transport);
+};
+logger.cleanup = cleanup;
+logger.logOutput = logOutput;
+logger.getLevel = () => winstonToZ2mLevel(transportsToUse[0].level);
+logger.setLevel = (level) => {
+    level = z2mToWinstonLevel(level);
+    logger.transports.forEach((transport) => transport.level = level);
+};
+
 // winston.config.syslog.levels doesnt have warn, but is required for syslog.
 /* istanbul ignore next */
 logger.warn = (message) => logger.warning(message);

lib/util/settings.js

@@ -1,691 +0,0 @@
-const data = require('./data');
-const utils = require('./utils');
-const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const yaml = require('./yaml');
-const Ajv = require('ajv');
-const ajv = new Ajv({allErrors: true});
-
-const defaults = {
-    whitelist: [],
-    ban: [],
-    permit_join: false,
-    mqtt: {
-        include_device_information: false,
-    },
-    serial: {
-        disable_led: false,
-    },
-    device_options: {},
-    map_options: {
-        graphviz: {
-            colors: {
-                fill: {
-                    enddevice: '#fff8ce',
-                    coordinator: '#e04e5d',
-                    router: '#4ea3e0',
-                },
-                font: {
-                    coordinator: '#ffffff',
-                    router: '#ffffff',
-                    enddevice: '#000000',
-                },
-                line: {
-                    active: '#009900',
-                    inactive: '#994444',
-                },
-            },
-        },
-    },
-    experimental: {
-        // json or attribute or attribute_and_json
-        output: 'json',
-        new_api: false,
-    },
-    advanced: {
-        legacy_api: true,
-        log_rotation: true,
-        log_output: ['console', 'file'],
-        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
-        log_file: 'log.txt',
-        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
-        log_syslog: {},
-        soft_reset_timeout: 0,
-        pan_id: 0x1a62,
-        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
-        channel: 11,
-        baudrate: 115200,
-        rtscts: true,
-        adapter_concurrent: null,
-
-        // Availability timeout in seconds, disabled by default.
-        availability_timeout: 0,
-        availability_blacklist: [],
-        availability_whitelist: [],
-
-        /**
-         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
-         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
-         *
-         * Therefore zigbee2mqtt BY DEFAULT caches all values and resend it with every message.
-         * advanced.cache_state in configuration.yaml allows to configure this.
-         * https://www.zigbee2mqtt.io/configuration/configuration.html
-         */
-        cache_state: true,
-
-        /**
-         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
-         * "ISO_8601": ISO 8601 format
-         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
-         * "epoch": milliseconds elapsed since the UNIX epoch
-         * "disable": no last_seen attribute (default)
-         */
-        last_seen: 'disable',
-
-        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
-        elapsed: false,
-
-        /**
-         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
-         *
-         * Network key will serve as the encryption key of your network.
-         * Changing this will require you to repair your devices.
-         */
-        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
-
-        /**
-         * Enables reporting feature
-         */
-        report: false,
-
-        /**
-         * Home Assistant discovery topic
-         */
-        homeassistant_discovery_topic: 'homeassistant',
-
-        /**
-         * Home Assistant status topic
-         */
-        homeassistant_status_topic: 'hass/status',
-
-        /**
-         * Home Assistant legacy triggers, when enabled:
-         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
-         * - A 'sensor_action' and 'sensor_click' will be discoverd
-         */
-        homeassistant_legacy_triggers: true,
-
-        /**
-         * Configurable timestampFormat
-         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
-         */
-        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
-    },
-};
-
-const schema = {
-    type: 'object',
-    properties: {
-        device_options: {type: 'object'},
-        homeassistant: {type: 'boolean'},
-        permit_join: {type: 'boolean'},
-        mqtt: {
-            type: 'object',
-            properties: {
-                base_topic: {type: 'string'},
-                server: {type: 'string'},
-                keepalive: {type: 'number'},
-                ca: {type: 'string'},
-                key: {type: 'string'},
-                cert: {type: 'string'},
-                user: {type: 'string'},
-                password: {type: 'string'},
-                client_id: {type: 'string'},
-                reject_unauthorized: {type: 'boolean'},
-                include_device_information: {type: 'boolean'},
-                version: {type: 'number'},
-            },
-            required: ['base_topic', 'server'],
-        },
-        serial: {
-            type: 'object',
-            properties: {
-                port: {type: ['string', 'null']},
-                disable_led: {type: 'boolean'},
-                adapter: {type: 'string', enum: ['deconz', 'zstack']},
-            },
-        },
-        ban: {type: 'array', items: {type: 'string'}},
-        whitelist: {type: 'array', items: {type: 'string'}},
-        experimental: {
-            type: 'object',
-            properties: {
-                transmit_power: {type: 'number'},
-            },
-        },
-        advanced: {
-            type: 'object',
-            properties: {
-                legacy_api: {type: 'boolean'},
-                pan_id: {type: 'number'},
-                ext_pan_id: {type: 'array', items: {type: 'number'}},
-                channel: {type: 'number', minimum: 11, maximum: 26},
-                cache_state: {type: 'boolean'},
-                log_rotation: {type: 'boolean'},
-                log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
-                log_output: {type: 'array', items: {type: 'string'}},
-                log_directory: {type: 'string'},
-                log_file: {type: 'string'},
-                baudrate: {type: 'number'},
-                rtscts: {type: 'boolean'},
-                soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: ['array', 'string'], items: {type: 'number'}},
-                last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
-                elapsed: {type: 'boolean'},
-                availability_timeout: {type: 'number', minimum: 0},
-                availability_blacklist: {type: 'array', items: {type: 'string'}},
-                availability_whitelist: {type: 'array', items: {type: 'string'}},
-                report: {type: 'boolean'},
-                homeassistant_discovery_topic: {type: 'string'},
-                homeassistant_status_topic: {type: 'string'},
-                timestamp_format: {type: 'string'},
-                adapter_concurrent: {type: 'number'},
-            },
-        },
-        map_options: {
-            type: 'object',
-            properties: {
-                graphviz: {
-                    type: 'object',
-                    properties: {
-                        colors: {
-                            type: 'object',
-                            properties: {
-                                fill: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                font: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                line: {
-                                    type: 'object',
-                                    properties: {
-                                        active: {type: 'string'},
-                                        inactive: {type: 'string'},
-                                    },
-                                },
-                            },
-                        },
-                    },
-                },
-            },
-        },
-        devices: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^0x[\\d\\w]{16}$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        retention: {type: 'number'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-        groups: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^[\\w].*$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        devices: {type: 'array', items: {type: 'string'}},
-                        optimistic: {type: 'boolean'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-    },
-    required: ['homeassistant', 'permit_join', 'mqtt'],
-};
-
-
-let _settings;
-let _settingsWithDefaults;
-
-function write() {
-    const settings = get();
-    const toWrite = objectAssignDeep.noMutate(settings);
-
-    // Read settings to check if we have to split devices/groups into separate file.
-    const actual = yaml.read(file);
-
-    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
-    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
-        if (actual[path[0]] && actual[path[0]][path[1]]) {
-            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
-            if (match) {
-                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
-                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
-            }
-        }
-    }
-
-    if (typeof actual.devices === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
-
-    if (typeof actual.groups === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
-
-    yaml.writeIfChanged(file, toWrite);
-
-    _settings = read();
-    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
-}
-
-function validate() {
-    const validate = ajv.compile(schema);
-    const valid = validate(_settings);
-    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
-        _settings.advanced.network_key !== 'GENERATE') {
-        throw new Error(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
-    }
-
-    const postfixes = utils.getEndpointNames();
-
-    // Verify that all friendly names are unique
-    const names = [];
-    const check = (name) => {
-        if (names.includes(name)) throw new Error(`Duplicate friendly_name '${name}' found`);
-        if (postfixes.includes(name)) throw new Error(`Following friendly_name are not allowed: '${postfixes}'`);
-        if (name.match(/.*\/\d*$/)) throw new Error(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
-        if (name.includes('#') || name.includes('+')) {
-            throw new Error(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
-        }
-        names.push(name);
-    };
-
-    const settingsWithDefaults = getWithDefaults();
-    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
-    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
-
-    if (settingsWithDefaults.mqtt.version !== 5) {
-        for (const device of Object.values(settingsWithDefaults.devices)) {
-            if (device.retention) {
-                throw new Error('MQTT retention requires protocol version 5');
-            }
-        }
-    }
-
-    const checkAvailabilityList = (list, type) => {
-        list.forEach((e) => {
-            if (!getEntity(e)) {
-                throw new Error(`Non-existing entity '${e}' specified in '${type}'`);
-            }
-        });
-    };
-
-    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
-    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
-
-    return !valid ? validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`) : null;
-}
-
-function read() {
-    const s = yaml.read(file);
-
-    // Read !secret MQTT username and password if set
-    const interpetValue = (value) => {
-        const re = /!(.*) (.*)/g;
-        const match = re.exec(value);
-        if (match) {
-            const file = data.joinPath(`${match[1]}.yaml`);
-            const key = match[2];
-            return yaml.read(file)[key];
-        } else {
-            return value;
-        }
-    };
-
-    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
-        s.mqtt.user = interpetValue(s.mqtt.user);
-        s.mqtt.password = interpetValue(s.mqtt.password);
-    }
-
-    if (s.advanced && s.advanced.network_key) {
-        s.advanced.network_key = interpetValue(s.advanced.network_key);
-    }
-
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = yaml.readIfExists(file) || {};
-    }
-
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = yaml.readIfExists(file) || {};
-    }
-
-    return s;
-}
-
-function get() {
-    if (!_settings) {
-        _settings = read();
-    }
-
-    return _settings;
-}
-
-function getWithDefaults() {
-    if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
-    }
-
-    if (!_settingsWithDefaults.devices) {
-        _settingsWithDefaults.devices = {};
-    }
-
-    if (!_settingsWithDefaults.groups) {
-        _settingsWithDefaults.groups = {};
-    }
-
-    return _settingsWithDefaults;
-}
-
-function set(path, value) {
-    let settings = get();
-
-    for (let i = 0; i < path.length; i++) {
-        const key = path[i];
-        if (i === path.length - 1) {
-            settings[key] = value;
-        } else {
-            if (!settings[key]) {
-                settings[key] = {};
-            }
-
-            settings = settings[key];
-        }
-    }
-
-    write();
-}
-
-function getGroup(IDorName) {
-    const settings = getWithDefaults();
-    const byID = settings.groups[IDorName];
-    if (byID) {
-        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
-    }
-
-    for (const [ID, group] of Object.entries(settings.groups)) {
-        if (group.friendly_name === IDorName) {
-            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
-        }
-    }
-
-    return null;
-}
-
-function getGroups() {
-    const settings = getWithDefaults();
-    return Object.entries(settings.groups).map(([ID, group]) => {
-        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
-    });
-}
-
-function getGroupThrowIfNotExists(IDorName) {
-    const group = getGroup(IDorName);
-    if (!group) {
-        throw new Error(`Group '${IDorName}' does not exist`);
-    }
-
-    return group;
-}
-
-function getDevice(IDorName) {
-    const settings = getWithDefaults();
-    const byID = settings.devices[IDorName];
-    if (byID) {
-        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
-    }
-
-    for (const [ID, device] of Object.entries(settings.devices)) {
-        if (device.friendly_name === IDorName) {
-            return {...device, ID, friendlyName: device.friendly_name};
-        }
-    }
-
-    return null;
-}
-
-function getDeviceThrowIfNotExists(IDorName) {
-    const device = getDevice(IDorName);
-    if (!device) {
-        throw new Error(`Device '${IDorName}' does not exist`);
-    }
-
-    return device;
-}
-
-function getEntity(IDorName) {
-    const device = getDevice(IDorName);
-    if (device) {
-        return {...device, type: 'device'};
-    }
-
-    const group = getGroup(IDorName);
-    if (group) {
-        return {...group, type: 'group'};
-    }
-
-    return null;
-}
-
-function addDevice(ID) {
-    if (getDevice(ID)) {
-        throw new Error(`Device '${ID}' already exists`);
-    }
-
-    const settings = get();
-
-    if (!settings.devices) {
-        settings.devices = {};
-    }
-
-    settings.devices[ID] = {friendly_name: ID};
-    write();
-    return getDevice(ID);
-}
-
-function whitelistDevice(ID) {
-    const settings = get();
-    if (!settings.whitelist) {
-        settings.whitelist = [];
-    }
-
-    if (settings.whitelist.includes(ID)) {
-        throw new Error(`Device '${ID}' already whitelisted`);
-    }
-
-    settings.whitelist.push(ID);
-    write();
-}
-
-function banDevice(ID) {
-    const settings = get();
-    if (!settings.ban) {
-        settings.ban = [];
-    }
-
-    settings.ban.push(ID);
-    write();
-}
-
-function removeDevice(IDorName) {
-    const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
-    delete settings.devices[device.ID];
-    write();
-}
-
-function addGroup(name, ID=null) {
-    if (getGroup(name) || getDevice(name)) {
-        throw new Error(`friendly_name '${name}' is already in use`);
-    }
-
-    const settings = get();
-    if (!settings.groups) {
-        settings.groups = {};
-    }
-
-    if (ID == null) {
-        // look for free ID
-        ID = '1';
-        while (settings.groups.hasOwnProperty(ID)) {
-            ID = (Number.parseInt(ID) + 1).toString();
-        }
-    } else {
-        // ensure provided ID is not in use
-        ID = ID.toString();
-        if (settings.groups.hasOwnProperty(ID)) {
-            throw new Error(`group id '${ID}' is already in use`);
-        }
-    }
-
-    settings.groups[ID] = {friendly_name: name};
-    write();
-
-    return getGroup(ID);
-}
-
-function groupHasDevice(group, keys) {
-    for (const device of group.devices) {
-        const index = keys.indexOf(device);
-        if (index != -1) {
-            return keys[index];
-        }
-    }
-
-    return false;
-}
-
-function addDeviceToGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
-
-    const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!groupHasDevice(group, keys)) {
-        group.devices.push(keys[0]);
-        write();
-    }
-}
-
-function removeDeviceFromGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
-    const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    const key = groupHasDevice(group, keys);
-    if (key) {
-        group.devices = group.devices.filter((d) => d != key);
-        write();
-    }
-}
-
-function removeGroup(groupIDorName) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
-    delete settings.groups[groupID];
-    write();
-}
-
-function changeDeviceOptions(IDorName, newOptions) {
-    const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
-    objectAssignDeep(settings.devices[device.ID], newOptions);
-    write();
-}
-
-function changeFriendlyName(IDorName, newName) {
-    if (getGroup(newName) || getDevice(newName)) {
-        throw new Error(`friendly_name '${newName}' is already in use`);
-    }
-
-    const settings = get();
-    if (getDevice(IDorName)) {
-        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
-    } else if (getGroup(IDorName)) {
-        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
-    } else {
-        throw new Error(`Device or group '${IDorName}' does not exist`);
-    }
-
-    write();
-}
-
-module.exports = {
-    validate,
-    get: getWithDefaults,
-    set,
-    getDevice,
-    getGroup,
-    getGroups,
-    getEntity,
-    whitelistDevice,
-    banDevice,
-    addDevice,
-    removeDevice,
-    addGroup,
-    removeGroup,
-    addDeviceToGroup,
-    removeDeviceFromGroup,
-    changeDeviceOptions,
-    changeFriendlyName,
-
-    // For tests only
-    _write: write,
-    _reRead: () => {
-        _settings = read();
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
-    },
-    _getDefaults: () => defaults,
-};

lib/util/settings.schema.json

@@ -0,0 +1,767 @@
+{
+  "type": "object",
+  "properties": {
+    "device_options": {
+      "type": "object"
+    },
+    "homeassistant": {
+      "title": "Home Assistant integration",
+      "type": "boolean",
+      "description": "Home Assistant integration (MQTT discovery)",
+      "default": false
+    },
+    "permit_join": {
+      "type": "boolean",
+      "default": false,
+      "title": "Permit join",
+      "description": "Allow new devices to join (re-applied at restart)"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
+    "mqtt": {
+      "type": "object",
+      "title": "MQTT",
+      "properties": {
+        "base_topic": {
+          "type": "string",
+          "title": "Base topic",
+          "requiresRestart": true,
+          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
+          "examples": ["zigbee2mqtt"]
+        },
+        "server": {
+          "type": "string",
+          "title": "MQTT server",
+          "requiresRestart": true,
+          "description": "MQTT server URL (use mqtts:// for SSL/TLS connection)",
+          "examples": ["mqtt://localhost:1883"]
+        },
+        "keepalive": {
+          "type": "number",
+          "title": "Keepalive",
+          "requiresRestart": true,
+          "description": "MQTT keepalive in second",
+          "default": 60
+        },
+        "ca": {
+          "type": "string",
+          "title": "Certificate authority",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS certificate of CA used to sign server and client certificates",
+          "examples": ["/etc/ssl/mqtt-ca.crt"]
+        },
+        "key": {
+          "type": "string",
+          "title": "SSL/TLS key",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS key for client-authentication",
+          "examples": ["/etc/ssl/mqtt-client.key"]
+        },
+        "cert": {
+          "type": "string",
+          "title": "SSL/TLS certificate",
+          "description": "Absolute path to SSL/TLS certificate for client-authentication",
+          "requiresRestart": true,
+          "examples": ["/etc/ssl/mqtt-client.crt"]
+        },
+        "user": {
+          "type": "string",
+          "title": "User",
+          "requiresRestart": true,
+          "description": "MQTT server authentication user",
+          "examples": ["johnnysilverhand"]
+        },
+        "password": {
+          "type": "string",
+          "title": "Password",
+          "requiresRestart": true,
+          "description": "MQTT server authentication password",
+          "examples": ["ILOVEPELMENI"]
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "requiresRestart": true,
+          "description": "MQTT client ID",
+          "examples": ["MY_CLIENT_ID"]
+        },
+        "reject_unauthorized": {
+          "type": "boolean",
+          "title": "Reject unauthorized",
+          "requiresRestart": true,
+          "description": "Disable self-signed SSL certificate",
+          "default": true
+        },
+        "include_device_information": {
+          "type": "boolean",
+          "title": "Include device information",
+          "description": "Include device information to mqtt messages",
+          "default": false
+        },
+        "version": {
+          "type": ["number", "null"],
+          "title": "Version",
+          "requiresRestart": true,
+          "description": "MQTT protocol version",
+          "default": 4,
+          "examples": [5]
+        },
+        "force_disable_retain": {
+          "type": "boolean",
+          "title": "Force disable retain",
+          "requiresRestart": true,
+          "description": "Disable retain for all send messages. ONLY enable if you MQTT broker doesn't support retained message (e.g. AWS IoT core, Azure IoT Hub, Google Cloud IoT core, IBM Watson IoT Platform). Enabling will break the Home Assistant integration",
+          "default": false
+        }
+      },
+      "required": ["base_topic", "server"]
+    },
+    "serial": {
+      "type": "object",
+      "title": "Serial",
+      "properties": {
+        "port": {
+          "type": ["string", "null"],
+          "title": "Port",
+          "requiresRestart": true,
+          "description": "Location of the adapter. To autodetect the port, set null",
+          "examples": ["/dev/ttyACM0"]
+        },
+        "disable_led": {
+          "type": "boolean",
+          "title": "Disable led",
+          "requiresRestart": true,
+          "description": "Disable LED of the adapter if supported",
+          "default": false
+        },
+        "adapter": {
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
+          "title": "Adapter",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Adapter type, not needed unless you are experiencing problems"
+        }
+      }
+    },
+    "blocklist": {
+      "title": "Blocklist",
+      "requiresRestart": true,
+      "description": "Block devices from the network (by ieeeAddr)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "passlist": {
+      "title": "Passlist",
+      "requiresRestart": true,
+      "description": "Allow only certain devices to join the network (by ieeeAddr). Note that all devices not on the passlist will be removed from the network!",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
+      "properties": {
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to supress logging"
+        },
+        "log_directory": {
+          "type": "string",
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
+        },
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "availability_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "default": 0,
+          "requiresRestart": true,
+          "title": "Availability Timeout",
+          "description": "Availability timeout in seconds when enabled, devices will be checked if they are still online. Only AC powered routers are checked for availability"
+        },
+        "availability_blocklist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability Blocklist",
+          "description": "Prevent devices from being checked for availability"
+        },
+        "availability_passlist": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "requiresRestart": true,
+          "title": "Availability passlist",
+          "description": "Only enable availability check for certain devices"
+        },
+        "availability_blacklist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "title": "Availability blacklist (deprecated, use availability_blocklist)",
+          "items": {
+            "type": "string"
+          }
+        },
+        "availability_whitelist": {
+          "type": "array",
+          "readOnly": true,
+          "requiresRestart": true,
+          "items": {
+            "type": "string"
+          },
+          "title": "Availability whitelist (deprecated, use passlist)"
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+          "default": true
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        }
+      }
+    },
+    "map_options": {
+      "type": "object",
+      "title": "Networkmap",
+      "properties": {
+        "graphviz": {
+          "type": "object",
+          "properties": {
+            "colors": {
+              "type": "object",
+              "properties": {
+                "fill": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "font": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "line": {
+                  "type": "object",
+                  "properties": {
+                    "active": {
+                      "type": "string"
+                    },
+                    "inactive": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "ota": {
+      "type": "object",
+      "title": "OTA updates",
+      "properties": {
+        "update_check_interval": {
+          "type": "number",
+          "title": "Update check interval",
+          "description": "Your device may request a check for a new firmware update. This value determines how frequently third party servers may actually be contacted to look for firmware updates. The value is set in minutes, and the default is 1 day.",
+          "default": 1440
+        },
+        "disable_automatic_update_check": {
+          "type": "boolean",
+          "title": "Disable automatic update check",
+          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
+          "default": false
+        }
+      }
+    },
+    "devices": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^0x[\\d\\w]{16}$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/device"
+        }
+      }
+    },
+    "groups": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[\\w].*$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/group"
+        }
+      }
+    },
+    "frontend": {
+      "type": "object",
+      "title": "Frontend",
+      "properties": {
+        "port": {
+          "type": "number",
+          "title": "Port",
+          "description": "Frontend binding port",
+          "default": 8080,
+          "requiresRestart": true
+        },
+        "host": {
+          "type": "string",
+          "title": "Bind host",
+          "description": "Frontend binding host",
+          "default":" 0.0.0.0",
+          "requiresRestart": true
+        },
+        "auth_token": {
+          "type": ["string", "null"],
+          "title": "Auth token",
+          "description": "Enables authentication, disabled by default",
+          "requiresRestart": true
+        }
+      }
+    }
+  },
+  "required": ["mqtt"],
+  "definitions": {
+    "device": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string",
+          "title": "Friendly name",
+          "description": "Used in the MQTT topic of a device. By default this is the device ID",
+          "readOnly": true
+        },
+        "retain": {
+          "type": "boolean",
+          "title": "Retain",
+          "description": "Retain MQTT messages of this device"
+        },
+        "retention": {
+          "type": "number",
+          "title": "Retention",
+          "description": "Sets the MQTT Message Expiry in seconds, Make sure to set mqtt.version to 5"
+        },
+        "qos": {
+          "type": "number",
+          "title": "QoS",
+          "description": "QoS level for MQTT messages of this device"
+        },
+        "debounce": {
+          "type": "number",
+          "title": "Debounce",
+          "description": "Debounces messages of this device"
+        },
+        "debounce_ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["action"],
+          "title": "Ignore debounce",
+          "description": "Protects unique payload values of specified payload properties from overriding within debounce time"
+        },
+        "optimistic": {
+          "type": "boolean",
+          "title": "Optimistic",
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["temperature", "battery", "action"],
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
+        },
+        "icon": {
+          "type": "string",
+          "title": "Icon",
+          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+        }
+      },
+      "required": ["friendly_name"]
+    },
+    "group": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string"
+        },
+        "retain": {
+          "type": "boolean"
+        },
+        "devices": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "optimistic": {
+          "type": "boolean"
+        },
+        "qos": {
+          "type": "number"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["friendly_name"]
+    }
+  }
+}

lib/util/settings.ts

@@ -0,0 +1,708 @@
+import data from './data';
+import * as utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import * as yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
+
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: Settings = {
+    passlist: [],
+    blocklist: [],
+    // Deprecated: use block/passlist
+    whitelist: [],
+    ban: [],
+    permit_join: false,
+    mqtt: {
+        include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
+    },
+    serial: {
+        disable_led: false,
+    },
+    device_options: {},
+    map_options: {
+        graphviz: {
+            colors: {
+                fill: {
+                    enddevice: '#fff8ce',
+                    coordinator: '#e04e5d',
+                    router: '#4ea3e0',
+                },
+                font: {
+                    coordinator: '#ffffff',
+                    router: '#ffffff',
+                    enddevice: '#000000',
+                },
+                line: {
+                    active: '#009900',
+                    inactive: '#994444',
+                },
+            },
+        },
+    },
+    experimental: {
+        // json or attribute or attribute_and_json
+        output: 'json',
+    },
+    advanced: {
+        legacy_api: true,
+        log_rotation: true,
+        log_symlink_current: false,
+        log_output: ['console', 'file'],
+        log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
+        log_file: 'log.txt',
+        log_level: /* istanbul ignore next */ process.env.DEBUG ? 'debug' : 'info',
+        log_syslog: {},
+        soft_reset_timeout: 0,
+        pan_id: 0x1a62,
+        ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
+        channel: 11,
+        adapter_concurrent: null,
+        adapter_delay: null,
+
+        // Availability timeout in seconds, disabled by default.
+        availability_timeout: 0,
+        availability_blocklist: [],
+        availability_passlist: [],
+        // Deprecated, use block/passlist
+        availability_blacklist: [],
+        availability_whitelist: [],
+
+        /**
+         * Home Assistant requires ALL attributes to be present in ALL MQTT messages send by the device.
+         * https://community.home-assistant.io/t/missing-value-with-mqtt-only-last-data-set-is-shown/47070/9
+         *
+         * Therefore Zigbee2MQTT BY DEFAULT caches all values and resend it with every message.
+         * advanced.cache_state in configuration.yaml allows to configure this.
+         * https://www.zigbee2mqtt.io/configuration/configuration.html
+         */
+        cache_state: true,
+        cache_state_persistent: true,
+        cache_state_send_on_startup: true,
+
+        /**
+         * Add a last_seen attribute to mqtt messages, contains date/time of zigbee message arrival
+         * "ISO_8601": ISO 8601 format
+         * "ISO_8601_local": Local ISO 8601 format (instead of UTC-based)
+         * "epoch": milliseconds elapsed since the UNIX epoch
+         * "disable": no last_seen attribute (default)
+         */
+        last_seen: 'disable',
+
+        // Optional: Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg
+        elapsed: false,
+
+        /**
+         * https://github.com/Koenkk/zigbee2mqtt/issues/685#issuecomment-449112250
+         *
+         * Network key will serve as the encryption key of your network.
+         * Changing this will require you to repair your devices.
+         */
+        network_key: [1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 13],
+
+        /**
+         * Enables reporting feature
+         */
+        report: false,
+
+        /**
+         * Home Assistant discovery topic
+         */
+        homeassistant_discovery_topic: 'homeassistant',
+
+        /**
+         * Home Assistant status topic
+         */
+        homeassistant_status_topic: 'hass/status',
+
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
+        /**
+         * Home Assistant legacy triggers, when enabled:
+         * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
+         * - A 'sensor_action' and 'sensor_click' will be discoverd
+         */
+        homeassistant_legacy_triggers: true,
+
+        /**
+         * Configurable timestampFormat
+         * https://github.com/Koenkk/zigbee2mqtt/commit/44db557a0c83f419d66755d14e460cd78bd6204e
+         */
+        timestamp_format: 'YYYY-MM-DD HH:mm:ss',
+    },
+    ota: {
+        /**
+         * Minimal time delta in minutes between polling third party server for potential firmware updates
+         */
+        update_check_interval: 24 * 60,
+        /**
+         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
+         * If set to true, only a user-initiated update search will be possible.
+         */
+        disable_automatic_update_check: false,
+    },
+    external_converters: [],
+};
+
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
+
+function write(): void {
+    const settings = getInternalSettings();
+    /* eslint-disable-line */ // @ts-ignore
+    const toWrite = objectAssignDeep.noMutate(settings);
+
+    // Read settings to check if we have to split devices/groups into separate file.
+    const actual = yaml.read(file);
+
+    // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
+    for (const path of [
+        ['mqtt', 'user'],
+        ['mqtt', 'password'],
+        ['advanced', 'network_key'],
+        ['frontend', 'auth_token'],
+    ]) {
+        if (actual[path[0]] && actual[path[0]][path[1]]) {
+            const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
+            if (match) {
+                yaml.updateIfChanged(data.joinPath(`${match[1]}.yaml`), match[2], toWrite[path[0]][path[1]]);
+                toWrite[path[0]][path[1]] = actual[path[0]][path[1]];
+            }
+        }
+    }
+
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            /* eslint-disable-line */ // @ts-ignore
+            const content = objectAssignDeep.noMutate(settings[type]);
+
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
+
+    yaml.writeIfChanged(file, toWrite);
+
+    _settings = read();
+    /* eslint-disable-line */ // @ts-ignore
+    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
+}
+
+export function validate(): string[] {
+    try {
+        getInternalSettings();
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            return [
+                `Your YAML file: '${error.file}' is invalid ` +
+                `(use https://jsonformatter.org/yaml-validator to find and fix the issue)`,
+            ];
+        }
+
+        return [error.message];
+    }
+
+    if (!ajvSetting(_settings)) {
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
+    }
+
+    const errors = [];
+    if (_settings.advanced && _settings.advanced.network_key && typeof _settings.advanced.network_key === 'string' &&
+        _settings.advanced.network_key !== 'GENERATE') {
+        errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
+    }
+
+    if (_settings.advanced && _settings.advanced.pan_id && typeof _settings.advanced.pan_id === 'string' &&
+        _settings.advanced.pan_id !== 'GENERATE') {
+        errors.push(`advanced.pan_id: should be number or 'GENERATE' (is '${_settings.advanced.pan_id}')`);
+    }
+
+    // Verify that all friendly names are unique
+    const names: string[] = [];
+    const check = (name: string): void => {
+        if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
+        errors.push(...utils.validateFriendlyName(name));
+        names.push(name);
+    };
+
+    const settingsWithDefaults = get();
+    Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
+    Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
+
+    if (settingsWithDefaults.mqtt.version !== 5) {
+        for (const device of Object.values(settingsWithDefaults.devices)) {
+            if (device.retention) {
+                errors.push('MQTT retention requires protocol version 5');
+            }
+        }
+    }
+
+    const checkAvailabilityList = (list: string[], type: string): void => {
+        list.forEach((e) => {
+            if (!getEntity(e)) {
+                errors.push(`Non-existing entity '${e}' specified in '${type}'`);
+            }
+        });
+    };
+
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blacklist, 'availability_blacklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_whitelist, 'availability_whitelist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_blocklist, 'availability_blocklist');
+    checkAvailabilityList(settingsWithDefaults.advanced.availability_passlist, 'availability_passlist');
+
+    return errors;
+}
+
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
+
+    // Read !secret MQTT username and password if set
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
+        const re = /!(.*) (.*)/g;
+        const match = re.exec(value);
+        if (match) {
+            const file = data.joinPath(`${match[1]}.yaml`);
+            const key = match[2];
+            return yaml.read(file)[key];
+        } else {
+            return value;
+        }
+    };
+
+    if (s.mqtt?.user && s.mqtt?.password) {
+        s.mqtt.user = interpetValue(s.mqtt.user);
+        s.mqtt.password = interpetValue(s.mqtt.password);
+    }
+
+    if (s.advanced?.network_key) {
+        s.advanced.network_key = interpetValue(s.advanced.network_key);
+    }
+
+    if (s.frontend?.auth_token) {
+        s.frontend.auth_token = interpetValue(s.frontend.auth_token);
+    }
+
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
+
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
+
+    return s;
+}
+
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
+        Object.keys(obj).forEach((key) => {
+            if (key !== 'type') {
+                if (key !== 'properties' && obj[key]) {
+                    const type = (obj[key].type || 'object').toString();
+                    const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
+                    const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
+                    if (process.env[envVariableName]) {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
+                            acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
+                            return acc[val];
+                        }, settings);
+
+                        if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
+                            setting[key] = JSON.parse(process.env[envVariableName]);
+                        } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
+                            setting[key] = process.env[envVariableName] * 1;
+                        } else if (type.indexOf('boolean') >= 0) {
+                            setting[key] = process.env[envVariableName].toLowerCase() === 'true';
+                        } else {
+                            /* istanbul ignore else */
+                            if (type.indexOf('string') >= 0) {
+                                setting[key] = process.env[envVariableName];
+                            }
+                        }
+                    }
+                }
+
+                if (typeof obj[key] === 'object' && obj[key]) {
+                    const newPath = [...path];
+                    if (key !== 'properties') {
+                        newPath.push(key);
+                    }
+                    iterate(obj[key], newPath);
+                }
+            }
+        });
+    };
+    iterate(schema.properties, []);
+}
+
+function getInternalSettings(): Partial<Settings> {
+    if (!_settings) {
+        _settings = read();
+        applyEnvironmentVariables(_settings);
+    }
+
+    return _settings;
+}
+
+export function get(): Settings {
+    if (!_settingsWithDefaults) {
+        /* eslint-disable-line */ // @ts-ignore
+        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, getInternalSettings());
+    }
+
+    if (!_settingsWithDefaults.devices) {
+        _settingsWithDefaults.devices = {};
+    }
+
+    if (!_settingsWithDefaults.groups) {
+        _settingsWithDefaults.groups = {};
+    }
+
+    return _settingsWithDefaults;
+}
+
+export function set(path: string[], value: string | number | boolean | KeyValue): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
+
+    for (let i = 0; i < path.length; i++) {
+        const key = path[i];
+        if (i === path.length - 1) {
+            settings[key] = value;
+        } else {
+            if (!settings[key]) {
+                settings[key] = {};
+            }
+
+            settings = settings[key];
+        }
+    }
+
+    write();
+}
+
+export function apply(newSettings: Record<string, unknown>): boolean {
+    ajvSetting(newSettings);
+    const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
+    if (errors.length) {
+        const error = errors[0];
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
+    }
+
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
+    _settings = objectAssignDeep.noMutate(_settings, newSettings);
+    write();
+
+    ajvRestartRequired(newSettings);
+    const restartRequired = ajvRestartRequired.errors &&
+        !!ajvRestartRequired.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
+}
+
+export function getGroup(IDorName: string): GroupSettings {
+    const settings = get();
+    const byID = settings.groups[IDorName];
+    if (byID) {
+        return {devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, group] of Object.entries(settings.groups)) {
+        if (group.friendly_name === IDorName) {
+            return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+export function getGroups(): GroupSettings[] {
+    const settings = get();
+    return Object.entries(settings.groups).map(([ID, group]) => {
+        return {devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+    });
+}
+
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
+    const group = getGroup(IDorName);
+    if (!group) {
+        throw new Error(`Group '${IDorName}' does not exist`);
+    }
+
+    return group;
+}
+
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
+    const byID = settings.devices[IDorName];
+    if (byID) {
+        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+    }
+
+    for (const [ID, device] of Object.entries(settings.devices)) {
+        if (device.friendly_name === IDorName) {
+            return {...device, ID, friendlyName: device.friendly_name};
+        }
+    }
+
+    return null;
+}
+
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
+    const device = getDevice(IDorName);
+    if (!device) {
+        throw new Error(`Device '${IDorName}' does not exist`);
+    }
+
+    return device;
+}
+
+export function getEntity(IDorName: string): EntitySettings {
+    const device = getDevice(IDorName);
+    if (device) {
+        return {...device, type: 'device'};
+    }
+
+    const group = getGroup(IDorName);
+    if (group) {
+        return {...group, type: 'group'};
+    }
+
+    return null;
+}
+
+export function addDevice(ID: string): DeviceSettings {
+    if (getDevice(ID)) {
+        throw new Error(`Device '${ID}' already exists`);
+    }
+
+    const settings = getInternalSettings();
+
+    if (!settings.devices) {
+        settings.devices = {};
+    }
+
+    settings.devices[ID] = {friendly_name: ID};
+    write();
+    return getDevice(ID);
+}
+
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.whitelist) {
+        settings.whitelist = [];
+    }
+
+    if (settings.whitelist.includes(ID)) {
+        throw new Error(`Device '${ID}' already whitelisted`);
+    }
+
+    settings.whitelist.push(ID);
+    write();
+}
+
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.blocklist) {
+        settings.blocklist = [];
+    }
+
+    settings.blocklist.push(ID);
+    write();
+}
+
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
+    if (!settings.ban) {
+        settings.ban = [];
+    }
+
+    settings.ban.push(ID);
+    write();
+}
+
+export function removeDevice(IDorName: string): void {
+    const device = getDeviceThrowIfNotExists(IDorName);
+    const settings = getInternalSettings();
+    delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendlyName}|${device.ID})(/(\\d|${utils.endpointNames.join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
+    write();
+}
+
+export function addGroup(name: string, ID?: string): GroupSettings {
+    utils.validateFriendlyName(name, true);
+    if (getGroup(name) || getDevice(name)) {
+        throw new Error(`friendly_name '${name}' is already in use`);
+    }
+
+    const settings = getInternalSettings();
+    if (!settings.groups) {
+        settings.groups = {};
+    }
+
+    if (ID == null) {
+        // look for free ID
+        ID = '1';
+        while (settings.groups.hasOwnProperty(ID)) {
+            ID = (Number.parseInt(ID) + 1).toString();
+        }
+    } else {
+        // ensure provided ID is not in use
+        ID = ID.toString();
+        if (settings.groups.hasOwnProperty(ID)) {
+            throw new Error(`Group ID '${ID}' is already in use`);
+        }
+    }
+
+    settings.groups[ID] = {friendly_name: name};
+    write();
+
+    return getGroup(ID);
+}
+
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
+    }
+
+    return null;
+}
+
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
+
+    const group = settings.groups[groupID];
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
+        group.devices.push(keys[0]);
+        write();
+    }
+}
+
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
+    const group = settings.groups[groupID];
+    if (!group.devices) {
+        return;
+    }
+
+    const key = groupGetDevice(group, keys);
+    if (key) {
+        group.devices = group.devices.filter((d) => d != key);
+        write();
+    }
+}
+
+export function removeGroup(IDorName: string): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
+    delete settings.groups[groupID];
+    write();
+}
+
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
+    delete newOptions.friendly_name;
+    delete newOptions.devices;
+    if (getDevice(IDorName)) {
+        objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.devices[getDevice(IDorName).ID]);
+    } else if (getGroup(IDorName)) {
+        objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.groups[getGroup(IDorName).ID]);
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
+}
+
+export function changeFriendlyName(IDorName: string, newName: string): void {
+    utils.validateFriendlyName(newName, true);
+    if (getGroup(newName) || getDevice(newName)) {
+        throw new Error(`friendly_name '${newName}' is already in use`);
+    }
+
+    const settings = getInternalSettings();
+    if (getDevice(IDorName)) {
+        settings.devices[getDevice(IDorName).ID].friendly_name = newName;
+    } else if (getGroup(IDorName)) {
+        settings.groups[getGroup(IDorName).ID].friendly_name = newName;
+    } else {
+        throw new Error(`Device or group '${IDorName}' does not exist`);
+    }
+
+    write();
+}
+
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
+
+export const testing = {
+    write,
+    clear: (): void => {
+        _settings = null;
+        _settingsWithDefaults = null;
+    },
+    defaults,
+};

lib/util/utils.js

@@ -1,153 +0,0 @@
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const equals = require('fast-deep-equal');
-
-// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
-const xiaomiManufacturerID = [4151, 4447];
-const ikeaTradfriManufacturerID = [4476];
-
-// construct a local ISO8601 string (instead of UTC-based)
-// Example:
-//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
-//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
-function toLocalISOString(dDate) {
-    const tzOffset = -dDate.getTimezoneOffset();
-    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
-    const pad = function(num) {
-        const norm = Math.floor(Math.abs(num));
-        return (norm < 10 ? '0' : '') + norm;
-    };
-
-    return dDate.getFullYear() +
-        '-' + pad(dDate.getMonth() + 1) +
-        '-' + pad(dDate.getDate()) +
-        'T' + pad(dDate.getHours()) +
-        ':' + pad(dDate.getMinutes()) +
-        ':' + pad(dDate.getSeconds()) +
-        plusOrMinus + pad(tzOffset / 60) +
-        ':' + pad(tzOffset % 60);
-}
-
-const endpointNames = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
-    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
-    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
-    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
-    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
-    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
-    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
-];
-
-function flatten(arr) {
-    return arr.reduce((flat, toFlatten) => {
-        return flat.concat(Array.isArray(toFlatten) ? flatten(toFlatten) : toFlatten);
-    }, []);
-}
-
-const forceEndDevice = flatten(
-    ['QBKG03LM', 'QBKG04LM', 'ZNMS13LM', 'ZNMS12LM']
-        .map((model) => zigbeeHerdsmanConverters.devices.find((d) => d.model === model))
-        .filter((definition) => definition.hasOwnProperty('zigbeeModel'))
-        .map((definition) => definition.zigbeeModel));
-
-async function getZigbee2mqttVersion() {
-    return new Promise((resolve, reject) => {
-        const git = require('git-last-commit');
-        const packageJSON = require('../../package.json');
-        const version = packageJSON.version;
-
-        git.getLastCommit((err, commit) => {
-            let commitHash = null;
-
-            if (err) {
-                try {
-                    commitHash = require('../../.hash.json').hash;
-                } catch (error) {
-                    commitHash = 'unknown';
-                }
-            } else {
-                commitHash = commit.shortHash;
-            }
-
-            resolve({commitHash, version});
-        });
-    });
-}
-
-function formatDate(date, type) {
-    let result;
-
-    switch (type) {
-    case 'ISO_8601':
-        result = new Date(date).toISOString();
-        break;
-    case 'ISO_8601_local':
-        result = toLocalISOString(new Date(date));
-        break;
-    case 'epoch':
-        result = date;
-        break;
-    default:
-        throw new Error(`Unsupported type '${type}'`);
-    }
-
-    return result;
-}
-
-function objectHasProperties(object, properties) {
-    for (const property of properties) {
-        if (!object.hasOwnProperty(property)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function equalsPartial(object, expected) {
-    for (const [key, value] of Object.entries(expected)) {
-        if (!equals(object[key], value)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function getObjectsProperty(objects, key, defaultValue) {
-    for (const object of objects) {
-        if (object.hasOwnProperty(key)) {
-            return object[key];
-        }
-    }
-
-    return defaultValue;
-}
-
-function getResponse(request, data, error) {
-    const response = {data, status: error ? 'error' : 'ok'};
-    if (error) response.error = error;
-    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
-        response.transaction = request.transaction;
-    }
-    return response;
-}
-
-module.exports = {
-    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
-    secondsToMilliseconds: (seconds) => seconds * 1000,
-    getZigbee2mqttVersion,
-    objectHasProperties,
-    getObjectsProperty,
-    getEndpointNames: () => endpointNames,
-    isXiaomiDevice: (device) => {
-        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
-            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
-    isRouter: (device) => device.type === 'Router' && !forceEndDevice.includes(device.modelID),
-    isBatteryPowered: (device) => device.powerSource && device.powerSource === 'Battery',
-    formatDate: (date, type) => formatDate(date, type),
-    equalsPartial,
-    getResponse,
-};

lib/util/utils.ts

@@ -0,0 +1,289 @@
+import equals from 'fast-deep-equal/es6';
+import humanizeDuration from 'humanize-duration';
+import data from './data';
+import vm from 'vm';
+import fs from 'fs';
+import path from 'path';
+
+// construct a local ISO8601 string (instead of UTC-based)
+// Example:
+//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
+//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
+function toLocalISOString(date: Date): string {
+    const tzOffset = -date.getTimezoneOffset();
+    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
+    const pad = (num: number): string => {
+        const norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+    };
+
+    return date.getFullYear() +
+        '-' + pad(date.getMonth() + 1) +
+        '-' + pad(date.getDate()) +
+        'T' + pad(date.getHours()) +
+        ':' + pad(date.getMinutes()) +
+        ':' + pad(date.getSeconds()) +
+        plusOrMinus + pad(tzOffset / 60) +
+        ':' + pad(tzOffset % 60);
+}
+
+export const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay', 'usb',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool', 'water', 'meter', 'wifi',
+];
+
+export function capitalize(s: string): string {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
+export async function getZigbee2MQTTVersion(): Promise<{commitHash: string, version: string}> {
+    const git = await import('git-last-commit');
+    const packageJSON = await import('../..' + '/package.json');
+
+    return new Promise((resolve) => {
+        const version = packageJSON.version;
+
+        git.getLastCommit((err: Error, commit: {shortHash: string}) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
+                } catch (error) {
+                    /* istanbul ignore next */
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
+            }
+
+            resolve({commitHash, version});
+        });
+    });
+}
+
+export async function getDependencyVersion(depend: string): Promise<{version: string}> {
+    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const version = packageJSON.version;
+    return {version};
+}
+
+export function formatDate(time: number, type: 'ISO_8601' | 'ISO_8601_local' | 'epoch' | 'relative'): string | number {
+    if (type === 'ISO_8601') return new Date(time).toISOString();
+    else if (type === 'ISO_8601_local') return toLocalISOString(new Date(time));
+    else if (type === 'epoch') return time;
+    else { // relative
+        return humanizeDuration(Date.now() - time, {language: 'en', largest: 2, round: true}) + ' ago';
+    }
+}
+
+export function objectHasProperties(object: {[s: string]: unknown}, properties: string[]): boolean {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function equalsPartial(object: {[s: string]: unknown}, expected: {[s: string]: unknown}): boolean {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+export function getObjectProperty(object: {[s: string]: unknown}, key: string, defaultValue: unknown): unknown {
+    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
+}
+
+export function getResponse(request: {transaction: string}, data: unknown, error: string):
+    {data: unknown, status: string, error?: string, transaction?: string} {
+    const response: {data: unknown, status: string, error?: string, transaction?: string} =
+        {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+export function parseJSON(value: string, failedReturnValue: unknown): unknown {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return failedReturnValue;
+    }
+}
+
+export function loadModuleFromText(moduleCode: string): unknown {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        clearInterval,
+        setImmediate,
+        clearImmediate,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    /* eslint-disable-line */ // @ts-ignore
+    return sandbox.module.exports;
+}
+
+export function loadModuleFromFile(modulePath: string): unknown {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
+/* eslint-disable-next-line */
+export function* getExternalConvertersDefinitions(settings: any): any {
+    const externalConverters = settings.get().external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converter;
+
+        if (moduleName.endsWith('.js')) {
+            converter = loadModuleFromFile(data.joinPath(moduleName));
+        } else {
+            converter = require(moduleName);
+        }
+
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+export function removeNullPropertiesFromObject(obj: KeyValue): void {
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value == null) {
+            delete obj[key];
+        } else if (typeof value === 'object') {
+            removeNullPropertiesFromObject(value);
+        }
+    }
+}
+
+export function getKey(
+    object: KeyValue, value: unknown, fallback: unknown, convertTo: (v: unknown) => unknown): unknown {
+    for (const key in object) {
+        if (object[key]===value) {
+            return convertTo ? convertTo(key) : key;
+        }
+    }
+
+    return fallback;
+}
+
+export function toNetworkAddressHex(value: number): string {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
+// eslint-disable-next-line
+export function toSnakeCase(value: string | KeyValue): any {
+    if (typeof value === 'object') {
+        value = {...value};
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
+export function validateFriendlyName(name: string, throwFirstError=false): string[] {
+    const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
+    return errors;
+}
+
+export function sleep(seconds: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+export function sanitizeImageParameter(parameter: string): string {
+    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
+    let sanitized = parameter;
+    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
+    return sanitized;
+}
+
+export function isAvailabilityEnabledForDevice(rd: ResolvedDevice, settings: Settings): boolean {
+    if (!settings.experimental.availability_new) return false;
+
+    if (rd.settings.hasOwnProperty('availability')) {
+        return !!rd.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(rd.name) || passlist.includes(rd.device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(rd.name) && !blocklist.includes(rd.device.ieeeAddr);
+}
+
+export function isXiaomiDevice(device: Device): boolean {
+    const xiaomiManufacturerID = [4151, 4447];
+    return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
+        (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
+}
+
+export function isIkeaTradfriDevice(device: Device): boolean {
+    return [4476].includes(device.manufacturerID);
+}
+
+export const hours = (hours: number): number => 1000 * 60 * 60 * hours;
+export const minutes = (minutes: number): number => 1000 * 60 * minutes;
+export const seconds = (seconds: number): number => 1000 * seconds;

lib/util/yaml.js

@@ -1,44 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-const equals = require('fast-deep-equal');
-
-function read(file) {
-    try {
-        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-    } catch (error) {
-        if (error.name === 'YAMLException') {
-            error.message =
-                `\n\n\n` +
-                `\t====================================================================\n` +
-                `\tYour YAML file '${file}' is invalid\n` +
-                `\tUse e.g. https://jsonformatter.org/yaml-validator to find and fix the issue.\n` +
-                `\t====================================================================\n` +
-                `\n\n` +
-                error.message;
-        }
-
-
-        throw error;
-    }
-}
-
-function readIfExists(file) {
-    return fs.existsSync(file) ? read(file) : null;
-}
-
-function writeIfChanged(file, content) {
-    const before = readIfExists(file);
-    if (!equals(before, content)) {
-        fs.writeFileSync(file, yaml.safeDump(content));
-    }
-}
-
-function updateIfChanged(file, key, value) {
-    const content = read(file);
-    if (content[key] !== value) {
-        content[key] = value;
-        writeIfChanged(file, content);
-    }
-}
-
-module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/yaml.ts

@@ -0,0 +1,35 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+
+export function read(file: string): KeyValue {
+    try {
+        /* eslint-disable-line */ // @ts-ignore
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+export function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+export function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+export function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}

lib/zigbee.js

@@ -7,8 +7,8 @@ const utils = require('./util/utils');
 const events = require('events');
 const objectAssignDeep = require('object-assign-deep');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify-without-jsonify');
 
-const endpointNames = utils.getEndpointNames();
 const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
 
 class Zigbee extends events.EventEmitter {
@@ -18,7 +18,8 @@ class Zigbee extends events.EventEmitter {
     }
 
     async start() {
-        logger.info(`Starting zigbee-herdsman...`);
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
         const herdsmanSettings = {
             network: {
                 panID: settings.get().advanced.pan_id,
@@ -37,12 +38,14 @@ class Zigbee extends events.EventEmitter {
             },
             adapter: {
                 concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
+                disableLED: settings.get().serial.disable_led,
             },
         };
 
         const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
         herdsmanSettingsLog.network.networkKey = 'HIDDEN';
-        logger.debug(`Using zigbee-herdsman with settings: '${JSON.stringify(herdsmanSettingsLog)}'`);
+        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
 
         if (herdsmanSettings.network.networkKey === 'GENERATE') {
             const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
@@ -50,35 +53,48 @@ class Zigbee extends events.EventEmitter {
             herdsmanSettings.network.networkKey = newKey;
         }
 
+        if (herdsmanSettings.network.panID === 'GENERATE') {
+            const newPanID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
+            settings.set(['advanced', 'pan_id'], newPanID);
+            herdsmanSettings.network.panID = newPanID;
+        }
+
+        let startResult;
         try {
             herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
-            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
-            await this.herdsman.start();
+            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings, logger);
+            startResult = await this.herdsman.start();
         } catch (error) {
             logger.error(`Error while starting zigbee-herdsman`);
             throw error;
         }
 
         this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
+        this.herdsman.on('deviceNetworkAddressChanged', (data) =>
+            this.emit('event', 'deviceNetworkAddressChanged', data));
         this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
         this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
         this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
         this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
         this.herdsman.on('message', (data) => this.emit('event', 'message', data));
+        this.herdsman.on('permitJoinChanged', (data) => this.emit('permitJoinChanged', data));
+        this.herdsman.on('lastSeenChanged', (data) => this.emit('lastSeenChanged', data));
 
-        logger.info('zigbee-herdsman started');
-        logger.info(`Coordinator firmware version: '${JSON.stringify(await this.getCoordinatorVersion())}'`);
-        logger.debug(`Zigbee network parameters: ${JSON.stringify(await this.herdsman.getNetworkParameters())}`);
+        logger.info(`zigbee-herdsman started (${startResult})`);
+        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
 
         for (const device of this.getClients()) {
-            // If a whitelist is used, all other device will be removed from the network.
-            if (settings.get().whitelist.length > 0) {
-                if (!settings.get().whitelist.includes(device.ieeeAddr)) {
-                    logger.warn(`Blacklisted device is connected (${device.ieeeAddr}), removing...`);
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
                     device.removeFromNetwork();
                 }
-            } else if (settings.get().ban.includes(device.ieeeAddr)) {
-                logger.warn(`Banned device is connected (${device.ieeeAddr}), removing...`);
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
                 device.removeFromNetwork();
             }
         }
@@ -94,33 +110,52 @@ class Zigbee extends events.EventEmitter {
             await this.herdsman.setTransmitPower(transmitPower);
             logger.info(`Set transmit power to '${transmitPower}'`);
         }
+
+        return startResult;
     }
 
     async getCoordinatorVersion() {
         return this.herdsman.getCoordinatorVersion();
     }
 
+    isStopping() {
+        return this.herdsman.isStopping();
+    }
+
+    async getNetworkParameters() {
+        return this.herdsman.getNetworkParameters();
+    }
+
     async reset(type) {
         await this.herdsman.reset(type);
     }
 
     async stop() {
+        logger.info('Stopping zigbee-herdsman...');
         await this.herdsman.stop();
-        logger.info('zigbee-herdsman stopped');
+        logger.info('Stopped zigbee-herdsman');
     }
 
-    async permitJoin(permit) {
+    async permitJoin(permit, resolvedEntity, time=undefined) {
         permit ?
-            logger.info('Zigbee: allowing new devices to join.') :
+            logger.info(`Zigbee: allowing new devices to join${resolvedEntity ? ` via ${resolvedEntity.name}` : ''}.`) :
             logger.info('Zigbee: disabling joining new devices.');
 
-        await this.herdsman.permitJoin(permit);
+        if (resolvedEntity && permit) {
+            await this.herdsman.permitJoin(permit, resolvedEntity.device, time);
+        } else {
+            await this.herdsman.permitJoin(permit, undefined, time);
+        }
     }
 
-    async getPermitJoin() {
+    getPermitJoin() {
         return this.herdsman.getPermitJoin();
     }
 
+    getPermitJoinTimeout() {
+        return this.herdsman.getPermitJoinTimeout();
+    }
+
     getClients() {
         return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
     }
@@ -133,6 +168,10 @@ class Zigbee extends events.EventEmitter {
         return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
     }
 
+    getDeviceByNetworkAddress(networkAddress) {
+        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
+    }
+
     getDevicesByType(type) {
         return this.herdsman.getDevicesByType(type);
     }
@@ -151,10 +190,17 @@ class Zigbee extends events.EventEmitter {
     resolveEntity(key) {
         assert(
             typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device' || key.constructor.name === 'Group',
+            key.constructor.name === 'Device' || key.constructor.name === 'Group' ||
+            key.constructor.name === 'Endpoint',
             `Wrong type '${typeof key}'`,
         );
 
+        const getEndpointName = (endpointNames, endpoint) => {
+            return endpoint ?
+                utils.getKey(endpointNames, endpoint.ID, null, ((v) => v === 'default' ? null : v)) : null;
+        };
+
+        const deviceOptions = settings.get().device_options;
         if (typeof key === 'string' || typeof key === 'number') {
             if (typeof key === 'number') {
                 key = key.toString();
@@ -168,10 +214,11 @@ class Zigbee extends events.EventEmitter {
                     endpoint: coordinator.getEndpoint(1),
                     settings: {friendlyName: 'Coordinator'},
                     name: 'Coordinator',
+                    endpointName: null,
                 };
             }
 
-            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
+            let endpointKey = utils.endpointNames.find((p) => key.endsWith(`/${p}`));
             const endpointByNumber = key.match(keyEndpointByNumber);
             if (!endpointKey && endpointByNumber) {
                 endpointKey = Number(endpointByNumber[1]);
@@ -190,48 +237,58 @@ class Zigbee extends events.EventEmitter {
                 }
 
                 const definition = zigbeeHerdsmanConverters.findByDevice(device);
-                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
+                const endpointNames = definition && definition.endpoint ? definition.endpoint(device) : null;
                 let endpoint;
                 if (endpointKey) {
                     if (endpointByNumber) {
                         endpoint = device.getEndpoint(endpointKey);
                     } else {
                         assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
-                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
-                        const endpointID = endpoints[endpointKey];
+                        assert(endpointNames != null,
+                            `Endpoint name '${endpointKey}' is given but no endpoints defined`);
+                        const endpointID = endpointNames[endpointKey];
                         assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
                         endpoint = device.getEndpoint(endpointID);
                     }
-                } else if (endpoints && endpoints['default']) {
-                    endpoint = device.getEndpoint(endpoints['default']);
+                } else if (endpointNames && endpointNames['default']) {
+                    endpoint = device.getEndpoint(endpointNames['default']);
                 } else {
                     endpoint = device.endpoints[0];
                 }
 
                 return {
-                    type: 'device', device, endpoint, settings: entity, name: entity.friendlyName, definition,
+                    type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
+                    name: entity.friendlyName, definition,
+                    endpointName: getEndpointName(endpointNames, endpoint),
                 };
             } else {
                 let group = this.getGroupByID(entity.ID);
                 if (!group) group = this.createGroup(entity.ID);
-                return {type: 'group', group, settings: entity, name: entity.friendlyName};
+                return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
             }
-        } else if (key.constructor.name === 'Device') {
-            const setting = settings.getEntity(key.ieeeAddr);
+        } else if (key.constructor.name === 'Device' || key.constructor.name === 'Endpoint') {
+            const device = key.constructor.name === 'Endpoint' ? key.getDevice() : key;
+            const setting = settings.getEntity(device.ieeeAddr);
+            const definition = zigbeeHerdsmanConverters.findByDevice(device);
+            const name = setting ? setting.friendlyName :
+                (device.type === 'Coordinator' ? 'Coordinator' : device.ieeeAddr);
+            const endpointNames = definition && definition.endpoint ? definition.endpoint(device) : null;
+
+            let endpoint;
+            if (key.constructor.name === 'Endpoint') endpoint = key;
+            else if (endpointNames && endpointNames['default']) endpoint = device.getEndpoint(endpointNames['default']);
+            else endpoint = device.endpoints[0];
+
             return {
-                type: 'device',
-                device: key,
-                endpoint: key.endpoints[0],
-                settings: setting,
-                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
-                definition: zigbeeHerdsmanConverters.findByDevice(key),
+                type: 'device', definition, name, device, endpoint, settings: {...deviceOptions, ...(setting || {})},
+                endpointName: getEndpointName(endpointNames, endpoint),
             };
         } else { // Group
             const setting = settings.getEntity(key.groupID);
             return {
                 type: 'group',
                 group: key,
-                settings: setting,
+                settings: {...deviceOptions, ...(setting || {})},
                 name: setting ? setting.friendlyName : key.groupID,
             };
         }
@@ -250,21 +307,23 @@ class Zigbee extends events.EventEmitter {
     }
 
     acceptJoiningDeviceHandler(ieeeAddr) {
-        // If set whitelist devices, all other device will be rejected to join the network
-        if (settings.get().whitelist.length > 0) {
-            if (settings.get().whitelist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining whitelisted device '${ieeeAddr}'`);
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
                 return true;
             } else {
-                logger.info(`Rejecting joining non-whitelisted device '${ieeeAddr}'`);
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
                 return false;
             }
-        } else if (settings.get().ban.length > 0) {
-            if (settings.get().ban.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining banned device '${ieeeAddr}'`);
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
                 return false;
             } else {
-                logger.info(`Accepting joining non-banned device '${ieeeAddr}'`);
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
                 return true;
             }
         } else {
@@ -272,8 +331,20 @@ class Zigbee extends events.EventEmitter {
         }
     }
 
-    async touchlinkFactoryReset() {
-        return this.herdsman.touchlinkFactoryReset();
+    async touchlinkFactoryResetFirst() {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr, channel) {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr, channel) {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan() {
+        return this.herdsman.touchlinkScan();
     }
 }
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.14.0",
+  "version": "1.21.1",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^13 || ^14"
+    "node": "^10 || ^12 || ^14 || ^15 || ^16"
   },
   "keywords": [
     "xiaomi",
@@ -20,9 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "test-with-coverage": "jest test --coverage",
-    "eslint": "node_modules/.bin/eslint lib/",
+    "build": "tsc && node index.js writehash",
+    "build-watch": "tsc --watch",
+    "eslint": "eslint lib/ --max-warnings=0",
     "start": "node index.js",
+    "test-with-coverage": "jest test --coverage",
     "test": "jest test",
     "test-watch": "jest test --watch"
   },
@@ -33,28 +35,49 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "*",
-    "debounce": "*",
-    "fast-deep-equal": "*",
-    "git-last-commit": "*",
-    "js-yaml": "*",
-    "mkdir-recursive": "*",
-    "moment": "*",
-    "mqtt": "*",
-    "object-assign-deep": "*",
-    "rimraf": "*",
-    "semver": "*",
-    "winston": "*",
-    "winston-syslog": "*",
-    "zigbee-herdsman": "0.12.94",
-    "zigbee-herdsman-converters": "12.0.110"
+    "ajv": "^8.6.2",
+    "debounce": "^1.2.1",
+    "deep-object-diff": "^1.1.0",
+    "fast-deep-equal": "^3.1.3",
+    "finalhandler": "^1.1.2",
+    "git-last-commit": "^1.0.1",
+    "humanize-duration": "^3.27.0",
+    "js-yaml": "^4.1.0",
+    "json-stable-stringify-without-jsonify": "^1.0.1",
+    "mkdir-recursive": "^0.4.0",
+    "moment": "^2.29.1",
+    "mqtt": "4.2.8",
+    "object-assign-deep": "^0.4.0",
+    "rimraf": "^3.0.2",
+    "semver": "^7.3.5",
+    "serve-static": "^1.14.1",
+    "source-map-support": "^0.5.19",
+    "winston": "^3.3.3",
+    "winston-syslog": "^2.4.4",
+    "ws": "^8.2.1",
+    "zigbee-herdsman": "0.13.138",
+    "zigbee-herdsman-converters": "14.0.238",
+    "zigbee2mqtt-frontend": "0.6.0"
   },
   "devDependencies": {
-    "eslint": "*",
-    "eslint-config-google": "*",
-    "eslint-plugin-jest": "*",
-    "jest": "*",
-    "tmp": "*"
+    "@babel/core": "^7.15.0",
+    "@babel/preset-env": "^7.15.0",
+    "@babel/preset-typescript": "^7.15.0",
+    "@types/debounce": "^1.2.0",
+    "@types/humanize-duration": "^3.25.1",
+    "@types/jest": "^27.0.1",
+    "@types/js-yaml": "^4.0.3",
+    "@types/object-assign-deep": "^0.4.0",
+    "@types/ws": "^7.4.7",
+    "@typescript-eslint/eslint-plugin": "^4.29.3",
+    "@typescript-eslint/parser": "^4.29.3",
+    "babel-jest": "^27.1.0",
+    "eslint": "^7.32.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-jest": "^24.4.0",
+    "jest": "^27.1.0",
+    "tmp": "^0.2.1",
+    "typescript": "^4.4.2"
   },
   "jest": {
     "coverageThreshold": {
@@ -66,7 +89,11 @@
       }
     },
     "collectCoverageFrom": [
-      "lib/**"
+      "lib/**/*.js",
+      "lib/**/*.ts"
     ]
+  },
+  "bin": {
+    "zigbee2mqtt": "cli.js"
   }
 }

scripts/dumpHomeAssistantMapping.js

@@ -0,0 +1,35 @@
+const HomeAssistant = require('../lib/extension/homeassistant');
+const stringify = require('json-stable-stringify-without-jsonify');
+const homeassistant = new HomeAssistant(null, null, null, null, {on: () => {}});
+const mapping = Object.entries(homeassistant._getMapping());
+mapping.sort((a, b) => {
+    if (a[0] > b[0]) {
+        return -1;
+    }
+    if (b[0] > a[0]) {
+        return 1;
+    }
+    return 0;
+});
+
+for (const map of mapping) {
+    map[1].sort((a, b) => {
+        return JSON.stringify(a).localeCompare(JSON.stringify(b));
+    });
+
+    const toDelete = [];
+    for (const entry of map[1]) {
+        if (entry.type === 'light') {
+            if (entry.discovery_payload.brightness === false) delete entry.discovery_payload.brightness;
+            if (entry.discovery_payload.color_temp === false) delete entry.discovery_payload.color_temp;
+            if (entry.discovery_payload.xy === false) delete entry.discovery_payload.xy;
+            if (entry.discovery_payload.hs === false) delete entry.discovery_payload.hs;
+        } else if (entry.object_id === 'linkquality') {
+            toDelete.push(entry);
+        }
+    }
+
+    toDelete.forEach((d) => map[1].splice(map[1].indexOf(d), 1));
+
+    console.log(`${map[0]} - ${stringify(map[1])}`);
+}

scripts/install.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 function zigbee2mqtt-show-short-info {
-  echo "Setup for Zigbee2mqtt bridge."
+  echo "Setup for Zigbee2MQTT bridge."
 }
 
 function zigbee2mqtt-show-long-info {
-  echo "This script installs the Zigbee2mqtt bridge"
+  echo "This script installs the Zigbee2MQTT bridge"
 }
 
 function zigbee2mqtt-show-copyright-info {
@@ -20,7 +20,7 @@ if [ -z "${node}" ]; then #Installing NodeJS if not already installed.
   apt install -y nodejs
 fi
 
-echo "Cloning Zigbee2mqtt git repository"
+echo "Cloning Zigbee2MQTT git repository"
 git clone https://github.com/Koenkk/zigbee2mqtt.git /opt/zigbee2mqtt
 chown -R pi:pi /opt/zigbee2mqtt
 

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,89 @@
+const {ZnpCommandStatus, NvSystemIds} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        let deletedCount = 0;
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            let len;
+            const needOsal = !(this.version.product == ZnpVersion.zStack3x0 && id <= 7);
+            if (needOsal) {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength', {id: id});
+                len = lengthRes.payload['length'];
+            } else {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'nvLength',
+                    {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0});
+                len = lengthRes.payload['len'];
+            }
+            if (len != 0) {
+                console.log(`NVMEM item #${id} - deleting, size: ${len}`);
+                if (needOsal) {
+                    await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                        {id: id, len: len},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                } else {
+                    await this.znp.request(Subsystem.SYS, 'nvDelete',
+                        {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0},
+                        null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                }
+                deletedCount++;
+            }
+        }
+        console.log(`Clearing all NVMEM items finished, deleted ${deletedCount} items`);
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

test/assets/exampleExtension.js

@@ -1,6 +1,7 @@
 class Example {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.mqtt = mqtt;
+        this.mqtt.publish('example/extension', 'call from constructor')
     }
 
     onMQTTConnected() {

test/assets/mock-external-converter-multiple.js

@@ -0,0 +1,33 @@
+const homeassistantSwitch = {
+    type: 'switch',
+    object_id: 'switch',
+    discovery_payload: {
+        payload_off: 'OFF',
+        payload_on: 'ON',
+        value_template: '{{ value_json.state }}',
+        command_topic: true,
+    },
+};
+
+const mockDevices = [{
+    mock: 1,
+    model: 'external_converters_device_1',
+    homeassistant: [homeassistantSwitch],
+    zigbeeModel: ['external_converter_device_1'],
+    vendor: 'external_1',
+    description: 'external_1',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+}, {
+    mock: 2,
+    model: 'external_converters_device_2',
+    zigbeeModel: ['external_converter_device_2'],
+    vendor: 'external_2',
+    description: 'external_2',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+}];
+
+module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -0,0 +1,12 @@
+const mockDevice = {
+    mock: true,
+    zigbeeModel: ['external_converter_device'],
+    vendor: 'external',
+    model: 'external_converter_device',
+    description: 'external',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
+};
+
+module.exports = mockDevice;

test/availability.test.js

@@ -1,42 +1,46 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
-
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
-
-const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const mocks = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Availability', () => {
     let controller;
+    let extension;
 
-    function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Availability');
+        await controller.enableDisableExtension(true, 'Availability');
+        extension = controller.extensions.find((e) => e.constructor.name === 'Availability');
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
         jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10)
-        controller = new Controller();
-        mocksClear.forEach((m) => m.mockClear());
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
     });
 
-    afterEach(async () => {
-        await controller.stop();
-        await flushPromises();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['advanced', 'availability_timeout'], 10);
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     })
 
     it('Should publish availabilty on startup', async () => {
@@ -80,7 +84,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availabilty online and query state on reconnect', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
+        const device = zigbeeHerdsman.devices.E11_G13;
         const endpoint = device.getEndpoint(1);
         device.ping.mockImplementationOnce(() => {throw new Error('failed')});
         jest.advanceTimersByTime(11 * 1000);
@@ -89,16 +93,15 @@ describe('Availability', () => {
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
+            'zigbee2mqtt/bulb_enddevice/availability',
           'online',
           { retain: true, qos: 0 },
           expect.any(Function)
         );
 
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
     });
 
     it('Should fail gracefully when quering state after reconnect fails', async () => {
@@ -153,7 +156,7 @@ describe('Availability', () => {
 
     it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
         MQTT.publish.mockClear();
-        getExtension().state = {};
+        extension.state = {};
         const payload = {device: zigbeeHerdsman.devices.bulb_color};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
@@ -172,15 +175,14 @@ describe('Availability', () => {
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledTimes(2);
         expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
     });
 
     it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
         MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
+        const device = zigbeeHerdsman.devices.E11_G13;
         const endpoint = device.getEndpoint(1);
         endpoint.read.mockClear();
         await zigbeeHerdsman.events.deviceJoined({device});
@@ -197,28 +199,30 @@ describe('Availability', () => {
         expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
     });
 
-    it('Should not ping devices blacklisted by friendly name', async () => {
+    it('Should not ping devices on blocklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
+        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not ping devices blacklisted by IEEE address', async () => {
+    it('Should not ping devices on blacklist by friendly name', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
+        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
+        await resetExtension();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should not ping devices on blocklist by IEEE address', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
@@ -238,43 +242,41 @@ describe('Availability', () => {
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by friendly name if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
+        await resetExtension();
+        device.ping.mockClear();
+        jest.advanceTimersByTime(11 * 1000);
+        await flushPromises();
+        expect(device.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should ping devices whitelisted by IEEE address if availability_whitelist is set', async () => {
+    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
+        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
+        await resetExtension();
         device.ping.mockClear();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should not ping non-whitelisted devices if availability_whitelist is set', async () => {
+    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
         const device = zigbeeHerdsman.devices.bulb;
-        getExtension().state[device.ieeeAddr] = false;
-        settings.set(['advanced', 'availability_whitelist'], ['0x000b57fffec6a5b3'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
+        extension.state[device.ieeeAddr] = false;
+        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
+        await resetExtension();
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
         expect(device.ping).toHaveBeenCalledTimes(0);
@@ -284,7 +286,7 @@ describe('Availability', () => {
 
     it('Should not read when device has no modelID and reconnects', async () => {
         const device = zigbeeHerdsman.devices.nomodel;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -293,7 +295,7 @@ describe('Availability', () => {
 
     it('Should not read when device has is unsupported', async () => {
         const device = zigbeeHerdsman.devices.unsupported_router;
-        getExtension().state[device.ieeeAddr] = true;
+        extension.state[device.ieeeAddr] = true;
         const endpoint = device.getEndpoint(1);
         await zigbeeHerdsman.events.deviceAnnounce({device});
         await flushPromises();
@@ -314,7 +316,7 @@ describe('Availability', () => {
     });
 
     it('Should publish availability when end device joins', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const payload = {device};
         MQTT.publish.mockClear();
         await zigbeeHerdsman.events.deviceJoined(payload);
@@ -355,13 +357,9 @@ describe('Availability', () => {
 
     it('Should republish existing state on MQTT connected', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
-        await controller.stop();
-        await flushPromises();
         MQTT.publish.mockClear();
-        controller = new Controller();
-        getExtension().state[device.ieeeAddr] = false;
-        await controller.start();
-        await flushPromises();
+        extension.state[device.ieeeAddr] = false;
+        await extension.onMQTTConnected();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bulb_color/availability',
           'offline',
@@ -369,4 +367,45 @@ describe('Availability', () => {
           expect.any(Function)
         );
     });
+
+    it('Should clear retained availability topic when device is remove', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb_color');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device is renamed', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({"from": "bulb_color", "to": "bulb_color_new_name"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color/availability',
+            null,
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should clear retained availability topic when device does not exist', async () => {
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/not_existing_hahaha/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/not_existing_hahaha/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/0x000b57fffec6a5b3/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/0x000b57fffec6a5b3/availability', null, {retain: true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bulb_color/availability', 'offline');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+    });
 });

test/availabilityNew.test.ts

@@ -0,0 +1,277 @@
+import data from './stub/data';
+import logger from './stub/logger';
+import MQTT from './stub/mqtt';
+import zigbeeHerdsman from './stub/zigbeeHerdsman';
+
+import * as utils from '../lib/util/utils';
+import * as settings from '../lib/util/settings';
+import Controller from '../lib/controller';
+import flushPromises from './lib/flushPromises';
+
+const devices = zigbeeHerdsman.devices;
+const mocks = [MQTT.publish, logger.warn, logger.debug];
+
+zigbeeHerdsman.returnDevices.concat(
+    [devices.bulb_color.ieeeAddr, devices.bulb_color_2.ieeeAddr, devices.coordinator.ieeeAddr, devices.remote])
+
+describe('Availability', () => {
+    let controller;
+
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'AvailabilityNew');
+        await controller.enableDisableExtension(true, 'AvailabilityNew');
+    }
+
+    const advancedTime = async (value) => {
+        jest.setSystemTime(Date.now() + value);
+        jest.advanceTimersByTime(value);
+        await flushPromises();
+    }
+
+    beforeAll(async () => {
+        jest.spyOn(utils, 'sleep').mockImplementation(async (seconds: number) => {});
+        jest.useFakeTimers('modern');
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        jest.useFakeTimers('modern').setSystemTime(utils.minutes(1));
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['experimental', 'availability_new'], true);
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], false);
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+        // @ts-ignore
+        Object.values(devices).forEach((d) => d.ping.mockClear());
+    });
+
+    afterEach(async () => {
+        // @ts-ignore
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+    })
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should publish availabilty on startup for device where it is enabled for', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bulb_color_2/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish offline for active device when not seen for 10 minutes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Shouldnt do anything for a device when availability: false is set for device', async () => {
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2}); // Coverage satisfaction
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish offline for passive device when not seen for 25 hours', async () => {
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for active device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should ping again when first ping fails', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        devices.bulb_color.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        devices.bulb_color.lastSeen = Date.now() + utils.minutes(10);
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(2);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(2, false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for passive device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.hours(24));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.remote});
+
+        await advancedTime(utils.hours(25));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        devices.remote.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        await advancedTime(utils.hours(3));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should immediately mark device as online when it lastSeen changes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(15));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+
+        devices.bulb_color.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to change availability timeout via device options', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b3', 'availability'], {timeout: 40});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(17));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to change availability timeout via avaiability options', async () => {
+        settings.set(['availability'], {active: {timeout: 30}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should stop pinging device when it leaves', async () => {
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(9));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: devices.bulb_color.ieeeAddr});
+        await flushPromises();
+
+        await advancedTime(utils.minutes(3));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to be disabled', async () => {
+        settings.set(['availability'], false);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should retrieve device state when it reconnects', async () => {
+        MQTT.publish.mockClear();
+
+        const endpoint = devices.bulb_color.getEndpoint(1);
+        endpoint.read.mockClear();
+
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        await advancedTime(utils.seconds(1));
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+        await advancedTime(utils.seconds(2));
+
+        expect(endpoint.read).toHaveBeenCalledTimes(3);
+        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
+            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
+
+        // Should stop when one request fails
+        endpoint.read.mockClear();
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        endpoint.read.mockImplementationOnce(() => {throw new Error('')});
+        await advancedTime(utils.seconds(3));
+        expect(endpoint.read).toHaveBeenCalledTimes(1);
+    });
+
+    it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
+        settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Deprecated - should allow to pass certain devices via availability_passlist', async () => {
+        settings.set(['advanced', 'availability_passlist'], [devices.bulb_color_2.ieeeAddr]);
+        settings.changeEntityOptions(devices.bulb_color_2.ieeeAddr, {availability: null});
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Deprecated - should allow to enable via availability_timeout', async () => {
+        settings.set(['availability'], false);
+        settings.set(['advanced', 'availability_timeout'], 60);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+});

test/bind.test.js

@@ -4,37 +4,418 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
+const stringify = require('json-stable-stringify-without-jsonify');
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
 
 describe('Bind', () => {
     let controller;
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
+            endpoint.bind = jest.fn();
             endpoint.bind.mockClear();
             endpoint.unbind.mockClear();
         }
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Bind');
+        await controller.enableDisableExtension(true, 'Bind');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        zigbeeHerdsman.groups.group_1.members = [];
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        debounce.mockClear();
+        await resetExtension();
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should bind to device and configure reporting', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+
+        // Setup
+        const originalDeviceOutputClusters = device.getEndpoint(1).outputClusters;
+        device.getEndpoint(1).outputClusters = [...device.getEndpoint(1).outputClusters, 768];
+        const originalTargetBinds = target.binds;
+        target.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target.getClusterAttributeValue.mockImplementationOnce((cluster, value) =>  undefined);
+        mockClear(device);
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({transaction: "1234", from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(target.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorCapabilities' ]);
+        expect(endpoint.bind).toHaveBeenCalledTimes(4);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("lightingColorCtrl", target);
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1}]);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"transaction": "1234","data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl", "lightingColorCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).outputClusters = originalDeviceOutputClusters;
     });
 
-    it('Should bind', async () => {
+    it('Should bind only specifief clusters', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color', clusters: ["genOnOff"]}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log error when there is nothing to bind', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        logger.error.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'button'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"button","clusters":[],"failed":[]},"status":"error","error":"Nothing to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+
+        // setup
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+        const originalRemoteBinds = device.getEndpoint(1).binds;
+        device.getEndpoint(1).binds = [];
+        const originalTargetBinds = target.binds;
+        target.binds = [
+            {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'lightingColorCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}
+        ];
+
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        delete zigbeeHerdsman.devices.bulb_color.meta.configured;
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(undefined);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+
+        // Disable reporting
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1}]);
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).binds = originalRemoteBinds;
+    });
+
+    it('Should unbind coordinator', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.unbind.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'Coordinator'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"Coordinator","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to groups', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        // Should configure reproting for device added to group
+        target1Member.configureReporting.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb');
+        await flushPromises();
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+    });
+
+    it('Should unbind from group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from group with skip_disable_reporting=true', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set to false, we don't expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(0);
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should unbind from group with skip_disable_reporting=false', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: false}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set, we expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 65535, "minimumReportInterval": 5, "reportableChange": 1}])
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 65535, "minimumReportInterval": 0, "reportableChange": 0}])
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should bind to group by number', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: '1'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should log when bind fails', async () => {
+        logger.error.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        endpoint.bind.mockImplementation(() => {throw new Error('failed')});
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color","clusters":[],"failed":["genScenes","genOnOff","genLevelCtrl"]},"status":"error","error":"Failed to bind"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind from non default endpoints', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch_double/right","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind server clusters to client clusters', async () => {
+        const device = zigbeeHerdsman.devices.temperature_sensor;
+        const target = zigbeeHerdsman.devices.heating_actuator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'temperature_sensor', to: 'heating_actuator'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("msTemperatureMeasurement", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"temperature_sensor","to":"heating_actuator","clusters":["msTemperatureMeasurement"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should bind to default endpoint returned by endpoints()', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
+        const endpoint = device.getEndpoint(2);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote/ep2', to: 'wall_switch'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote/ep2","to":"wall_switch","clusters":["genOnOff"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from default_bind_group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = 'default_bind_group';
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: target}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"default_bind_group","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when source not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote_not_existing', to: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote_not_existing","to":"bulb_color"},"status":"error","error":"Source device 'remote_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error bind fails when target not existing', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"remote","to":"bulb_color_not_existing"},"status":"error","error":"Target device or group 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should bind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -45,7 +426,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -54,7 +434,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log error when there is nothing to bind', async () => {
+    it('Legacy api: Should log error when there is nothing to bind', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
@@ -65,7 +445,7 @@ describe('Bind', () => {
         expect(logger.error).toHaveBeenCalledWith(`Nothing to bind from 'remote' to 'button'`);
     });
 
-    it('Should unbind', async () => {
+    it('Legacy api: Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -76,7 +456,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -85,7 +464,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should unbind coordinator', async () => {
+    it('Legacy api: Should unbind coordinator', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
@@ -97,7 +476,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -106,7 +484,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to groups', async () => {
+    it('Legacy api: Should bind to groups', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -117,7 +495,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -126,7 +503,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should bind to group by number', async () => {
+    it('Legacy api: Should bind to group by number', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
         const endpoint = device.getEndpoint(1);
@@ -137,7 +514,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -146,7 +522,7 @@ describe('Bind', () => {
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genLevelCtrl'}});
     });
 
-    it('Should log when bind fails', async () => {
+    it('Legacy api: Should log when bind fails', async () => {
         logger.error.mockClear();
         const device = zigbeeHerdsman.devices.remote;
         const endpoint = device.getEndpoint(1);
@@ -154,12 +530,11 @@ describe('Bind', () => {
         endpoint.bind.mockImplementationOnce(() => {throw new Error('failed')});
         MQTT.events.message('zigbee2mqtt/bridge/bind/remote', 'bulb_color');
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Failed to bind cluster 'genScenes' from 'remote' to 'bulb_color' (Error: failed)");
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
     });
 
-    it('Should bind from non default endpoints', async () => {
+    it('Legacy api: Should bind from non default endpoints', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG03LM.getEndpoint(3);
         const endpoint = device.getEndpoint(2);
@@ -170,7 +545,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should bind to default endpoint returned by endpoints()', async () => {
+    it('Legacy api: Should bind to default endpoint returned by endpoints()', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
         const endpoint = device.getEndpoint(2);
@@ -181,7 +556,7 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
     });
 
-    it('Should unbind from default_bind_group', async () => {
+    it('Legacy api: Should unbind from default_bind_group', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = 'default_bind_group';
         const endpoint = device.getEndpoint(1);
@@ -192,7 +567,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -200,4 +574,37 @@ describe('Bind', () => {
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
     });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read.mockClear();
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(4);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
 });

test/configure.test.js

@@ -4,20 +4,22 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Configure', () => {
     let controller;
+    let coordinatorEndpoint;
 
-    expectRemoteConfigured = () => {
+    const expectRemoteConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(2);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
 
         const endpoint2 = device.getEndpoint(2);
         expect(endpoint2.write).toHaveBeenCalledTimes(1);
@@ -25,13 +27,27 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
-    expectRemoteNotConfigured = () => {
+    const expectBulbConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(2);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities']);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
+    }
+
+    const expectBulbNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(0);
+    }
+
+    const expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(0);
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -40,58 +56,127 @@ describe('Configure', () => {
         }
     }
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Configure');
+        await controller.enableDisableExtension(true, 'Configure');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
-        mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
-    it('Should configure on startup', async () => {
-        expectRemoteConfigured();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should configure Router on startup', async () => {
+        expectBulbConfigured();
+    });
+
+    it('Should not configure EndDevice on startup', async () => {
+        expectRemoteNotConfigured();
     });
 
     it('Should re-configure when device rejoins', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         await flushPromises();
         mockClear(device);
         const payload = {device};
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
+    });
+
+    it('Should reconfigure reporting on reportingDisabled event', async () => {
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
+        mockClear(device);
+        expectBulbNotConfigured();
+        controller.eventBus.emit('reportingDisabled', {device})
+        await flushPromises();
+        expectBulbConfigured();
     });
 
     it('Should not configure twice', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expectBulbNotConfigured();
     });
 
     it('Should configure on zigbee message when not configured yet', async () => {
-        const device = zigbeeHerdsman.devices.remote;
+        const device = zigbeeHerdsman.devices.bulb;
         delete device.meta.configured;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
-    it('Should allow to reconfigure manually', async () => {
+    it('Should allow to configure via MQTT', async () => {
+        mockClear(zigbeeHerdsman.devices.remote);
+        expectRemoteNotConfigured();
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
+        await flushPromises();
         expectRemoteConfigured();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device does not exist', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "not_existing_device"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "not_existing_device"},"status":"error","error": "Device 'not_existing_device' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when configure fails', async () => {
+        zigbeeHerdsman.devices.remote.getEndpoint(1).bind.mockImplementationOnce(async () => {throw new Error('Bind timeout after 10s')});
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "remote"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "remote"},"status":"error","error": "Failed to configure (Bind timeout after 10s)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Fail to configure via MQTT when device has no configure', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "0x90fd9ffffe4b64ax", transaction: 20}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/configure',
+            stringify({"data":{"id": "0x90fd9ffffe4b64ax"},"status":"error","error": "Device 'ZNLDP12LM' cannot be configured","transaction":20}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Legacy api: Should allow to reconfigure manually', async () => {
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote');
@@ -99,16 +184,16 @@ describe('Configure', () => {
         expectRemoteConfigured();
     });
 
-    it('Shouldnt manually reconfigure when device does not exist', async () => {
+    it('Legacy api: Shouldnt manually reconfigure when device does not exist', async () => {
         await MQTT.events.message('zigbee2mqtt/bridge/configure', 'remote_random_non_existing');
         await flushPromises();
         expect(logger.error).toHaveBeenCalledWith(`Device 'remote_random_non_existing' does not exist`);
     });
 
-    it('Should skip reconfigure when device does not require this', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+    it('Legacy api: Should skip reconfigure when device does not require this', async () => {
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x90fd9ffffe4b64ax');
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
     it('Should not configure when interviewing', async () => {

test/controller.test.js

@@ -3,13 +3,13 @@ const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const path = require('path');
-const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = require('./lib/flushPromises');
 const tmp = require('tmp');
 const mocksClear = [
-    zigbeeHerdsman.permitJoin, mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
     zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
 ];
@@ -18,32 +18,49 @@ const fs = require('fs');
 
 describe('Controller', () => {
     let controller;
+    let mockExit;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
 
     beforeEach(() => {
         zigbeeHerdsman.returnDevices.splice(0);
-        controller = new Controller();
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeDefaultState();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":115200,"rtscts":true,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
         expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
         expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
         expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
         expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}',{ retain: true, qos: 0 }, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', '{"brightness":255}', { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Start controller when permit join fails', async () => {
+        zigbeeHerdsman.permitJoin.mockImplementationOnce(() => {throw new Error("failed!")});
+        await controller.start();
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
     });
 
     it('Start controller with specific MQTT settings', async () => {
@@ -85,20 +102,32 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
-    it('Should generate network_key when set to GENERATE', async () => {
+    it('Should generate network_key and pan_id when set to GENERATE', async () => {
         settings.set(['advanced', 'network_key'], 'GENERATE');
+        settings.set(['advanced', 'pan_id'], 'GENERATE');
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.panID).toStrictEqual(expect.any(Number));
         expect(data.read().advanced.network_key.length).toStrictEqual(16);
+        expect(data.read().advanced.pan_id).toStrictEqual(expect.any(Number));
     });
 
     it('Start controller should publish cached states', async () => {
         data.writeDefaultState();
         await controller.start();
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", `{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}`, {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", `{"brightness":255}`, {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/remote", stringify({"brightness":255}), {"qos": 0, "retain": true}, expect.any(Function));
+    });
+
+    it('Start controller should not publish cached states when disabled', async () => {
+        settings.set(['advanced', 'cache_state_send_on_startup'], false);
+        data.writeDefaultState();
+        await controller.start();
+        await flushPromises();
+        const publishedTopics = MQTT.publish.mock.calls.map(m => m[0]);
+        expect(publishedTopics).toEqual(expect.not.arrayContaining(["zigbee2mqtt/bulb", "zigbee2mqtt/remote"]));
     });
 
     it('Start controller should not publish cached states when cache_state is false', async () => {
@@ -111,14 +140,12 @@ describe('Controller', () => {
     });
 
     it('Log when MQTT client is unavailable', async () => {
-        jest.useFakeTimers();
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
         controller.mqtt.client.reconnecting = false;
     });
@@ -132,7 +159,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(2);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
-        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"state\":\"ON\",\"brightness\":50,\"color_temp\":370,\"linkquality\":99,\"color\":{\"r\":100,\"g\":50,\"b\":10},\"dummy\":{\"1\":\"yes\",\"2\":\"no\"}}");
+        expect(logger.error).toHaveBeenCalledWith("Cannot send message: topic: 'zigbee2mqtt/bulb', payload: '{\"brightness\":50,\"color\":{\"b\":10,\"g\":50,\"r\":100},\"color_temp\":370,\"dummy\":{\"1\":\"yes\",\"2\":\"no\"},\"linkquality\":99,\"state\":\"ON\"}");
         controller.mqtt.client.reconnecting = false;
     });
 
@@ -143,6 +170,14 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
+    it('Should remove device not on passlist on startup', async () => {
+        settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
+    });
+
     it('Should remove non whitelisted devices on startup', async () => {
         settings.set(['whitelist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -151,6 +186,14 @@ describe('Controller', () => {
         expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(1);
     });
 
+    it('Should remove device on blocklist on startup', async () => {
+        settings.set(['blocklist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        await controller.start();
+        await flushPromises();
+        expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb.removeFromNetwork).toHaveBeenCalledTimes(0);
+    });
+
     it('Should remove banned devices on startup', async () => {
         settings.set(['ban'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
         await controller.start();
@@ -169,16 +212,7 @@ describe('Controller', () => {
         settings.set(['permit_join'], false);
         await controller.start();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
-    });
-
-    it('Refuse to start when configuration.yaml is invalid', async () => {
-        settings.set(['permit_join'], 'invalid');
-        await controller.start();
-        expect(logger.error).toHaveBeenCalledWith('Refusing to start, configuration.yaml is not valid, found the following errors:');
-        expect(logger.error).toHaveBeenCalledWith('\t - permit_join should be boolean');
-        expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
     });
 
     it('Start controller with disable_led', async () => {
@@ -201,7 +235,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(0);
+        expect(mockExit).toHaveBeenCalledWith(0, null);
     });
 
     it('Start controller and stop', async () => {
@@ -211,7 +245,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
     });
 
     it('Start controller adapter disconnects', async () => {
@@ -222,7 +256,7 @@ describe('Controller', () => {
         expect(MQTT.end).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.stop).toHaveBeenCalledTimes(1);
         expect(mockExit).toHaveBeenCalledTimes(1);
-        expect(mockExit).toHaveBeenCalledWith(1);
+        expect(mockExit).toHaveBeenCalledWith(1, null);
     });
 
     it('Handle mqtt message', async () => {
@@ -232,6 +266,17 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
     });
 
+    it('Skip MQTT messages on topic we published to', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'zigbee2mqtt/skip-this-topic' with data 'skipped'")
+        logger.debug.mockClear();
+        await controller.mqtt.publish('skip-this-topic', '', {});
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledTimes(0);
+    });
+
     it('On zigbee event message', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
@@ -273,51 +318,51 @@ describe('Controller', () => {
         const payload = {device};
         await zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('acceptJoiningDeviceHandler reject banned device', async () => {
+    it('acceptJoiningDeviceHandler reject device on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler accept not banned device', async () => {
+    it('acceptJoiningDeviceHandler accept device not on blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['ban'], ['123']);
+        settings.set(['blocklist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler accept device on passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler reject non-whitelisted device', async () => {
+    it('acceptJoiningDeviceHandler reject device not in passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], ['123']);
+        settings.set(['passlist'], ['123']);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(false);
     });
 
-    it('acceptJoiningDeviceHandler should prefer whitelist above ban', async () => {
+    it('acceptJoiningDeviceHandler should prefer passlist above blocklist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
-        settings.set(['whitelist'], [device.ieeeAddr]);
-        settings.set(['ban'], [device.ieeeAddr]);
+        settings.set(['passlist'], [device.ieeeAddr]);
+        settings.set(['blocklist'], [device.ieeeAddr]);
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
         expect(await handler(device.ieeeAddr)).toBe(true);
     });
 
-    it('acceptJoiningDeviceHandler accept when no ban and whitelist', async () => {
+    it('acceptJoiningDeviceHandler accept when not on blocklist and passlist', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
         const handler = zigbeeHerdsman.constructor.mock.calls[0][0].acceptJoiningDeviceHandler;
@@ -331,7 +376,7 @@ describe('Controller', () => {
         zigbeeHerdsman.events.deviceJoined(payload);
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_connected","message":{"friendly_name":"bulb"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_connected","message":{"friendly_name":"bulb"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee deviceInterview started', async () => {
@@ -340,7 +385,7 @@ describe('Controller', () => {
         const payload = {device, status: 'started'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_started","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview failed', async () => {
@@ -349,7 +394,7 @@ describe('Controller', () => {
         const payload = {device, status: 'failed'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_failed","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful supported', async () => {
@@ -358,7 +403,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"bulb","model":"LED1545G12","vendor":"IKEA","description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","supported":true}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee deviceInterview successful not supported', async () => {
@@ -367,7 +412,7 @@ describe('Controller', () => {
         const payload = {device, status: 'successful'};
         await zigbeeHerdsman.events.deviceInterview(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"pairing","message":"interview_successful","meta":{"friendly_name":"0x0017880104e45518","supported":false}}), { retain: false, qos: 0 }, expect.any(Function));
     });
 
     it('On zigbee event device announce', async () => {
@@ -377,7 +422,7 @@ describe('Controller', () => {
         await zigbeeHerdsman.events.deviceAnnounce(payload);
         await flushPromises();
         expect(logger.debug).toHaveBeenCalledWith(`Device 'bulb' announced itself`);
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_announced","message":"announce","meta":{"friendly_name":"bulb"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and settings)', async () => {
@@ -389,7 +434,7 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('On zigbee event device leave (removed from database and NOT settings)', async () => {
@@ -400,21 +445,23 @@ describe('Controller', () => {
         const payload = {ieeeAddr: device.ieeeAddr};
         await zigbeeHerdsman.events.deviceLeave(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', '{"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}', { retain: false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/log', stringify({"type":"device_removed","message":"left_network","meta":{"friendly_name":"0x000b57fffec6a5b2"}}), { retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Publish entity state attribute output', async () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute');
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        await controller.publishEntityState('bulb', {dummy: {1: 'yes', 2: 'no'}, color: {r: 100, g: 50, b: 10}, state: 'ON', test: undefined, test1: null, color_temp: 370, brightness: 50});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color", '100,50,10', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", 'yes', {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", 'no', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-1", "yes", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/dummy-2", "no", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test1", '', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/test", '', {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state attribute_json output', async () => {
@@ -428,7 +475,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/color_temp", "370", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/linkquality", "99", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
 
@@ -442,7 +489,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state attribute_json output filtered (device_options)', async () => {
@@ -455,7 +502,7 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "200", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"ON","brightness":200}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON","brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish entity state with device information', async () => {
@@ -464,12 +511,12 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null}}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
         await controller.publishEntityState('unsupported2', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', '{"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null}}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state without retain', async () => {
@@ -478,7 +525,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": false}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": false}, expect.any(Function));
     });
 
     it('Should publish entity state with retain', async () => {
@@ -487,7 +534,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Should publish entity state with expiring retention', async () => {
@@ -498,7 +545,7 @@ describe('Controller', () => {
         MQTT.publish.mockClear();
         await controller.publishEntityState('bulb', {state: 'ON'});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', '{"state":"ON","brightness":50,"color_temp":370,"linkquality":99}', {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
     });
 
     it('Publish entity state no empty messages', async () => {
@@ -510,6 +557,30 @@ describe('Controller', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should allow to disable state persistency', async () => {
+        settings.set(['advanced', 'cache_state_persistent'], false);
+        data.removeState();
+        await controller.start();
+        MQTT.publish.mockClear();
+        await controller.publishEntityState('bulb', {state: 'ON'});
+        await controller.publishEntityState('bulb', {brightness: 200});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON", brightness: 200}), {"qos": 0, "retain": true}, expect.any(Function));
+        await controller.stop();
+        expect(data.stateExists()).toBeFalsy();
+    });
+
+    it('Shouldnt crash when it cannot save state', async () => {
+        data.removeState();
+        await controller.start();
+        logger.error.mockClear();
+        controller.state.file = "/";
+        await controller.state.save();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to write state to '/' (EISDIR: illegal operation on a directory, open '/')`);
+    });
+
     it('Publish should not cache when set', async () => {
         settings.set(['advanced', 'cache_state'], false);
         data.writeEmptyState();
@@ -519,8 +590,8 @@ describe('Controller', () => {
         await controller.publishEntityState('bulb', {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"state\":\"ON\"}", {"qos": 0, "retain": true}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", "{\"brightness\":200}", {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON"}), {"qos": 0, "retain": true}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
     it('Publish should not do anything for unknown entity', async () => {
@@ -538,14 +609,33 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
-    it('Load user extension', async () => {
-        const extensionPath = path.join(data.mockDir, 'extension');
-        fs.mkdirSync(extensionPath);
-        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
-        controller = new Controller();
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.start();
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state" },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
 });

test/externalConverters.test.js

@@ -0,0 +1,139 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const flushPromises = require('./lib/flushPromises');
+const path = require('path');
+const fs = require('fs');
+
+zigbeeHerdsmanConverters.addDeviceDefinition = jest.fn();
+
+const mocksClear = [zigbeeHerdsmanConverters.addDeviceDefinition, zigbeeHerdsman.permitJoin,
+    mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+jest.mock(
+    'mock-external-converter-module', () => {
+        return {
+            mock: true
+        };
+    }, {
+        virtual: true
+    });
+
+jest.mock(
+    'mock-multiple-external-converter-module', () => {
+        return [{
+            mock: 1
+        }, {
+            mock: 2
+        }];
+    }, {
+        virtual: true
+    });
+
+describe('Loads external converters', () => {
+    let controller;
+
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'ExternalConverters');
+        await controller.enableDisableExtension(true, 'ExternalConverters');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    it('Does not load external converters', async () => {
+        settings.set(['external_converters'], []);
+        await resetExtension();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(0);
+    });
+
+    it('Loads external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        await resetExtension();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true,
+            zigbeeModel: ['external_converter_device'],
+            vendor: 'external',
+            model: 'external_converter_device',
+            description: 'external',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+    });
+
+    it('Loads multiple external converters', async () => {
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter-multiple.js'), path.join(data.mockDir, 'mock-external-converter-multiple.js'));
+        const devicesCount = zigbeeHerdsman.devices.lenght;
+        settings.set(['external_converters'], ['mock-external-converter-multiple.js']);
+        await resetExtension();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1,
+            model: 'external_converters_device_1',
+            zigbeeModel: ['external_converter_device_1'],
+            vendor: 'external_1',
+            description: 'external_1',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2,
+            model: 'external_converters_device_2',
+            zigbeeModel: ['external_converter_device_2'],
+            vendor: 'external_2',
+            description: 'external_2',
+            fromZigbee: [],
+            toZigbee: [],
+            exposes: []
+        });
+    });
+
+    it('Loads external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-external-converter-module']);
+        await resetExtension();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledWith({
+            mock: true
+        });
+    });
+
+    it('Loads multiple external converters from package', async () => {
+        settings.set(['external_converters'], ['mock-multiple-external-converter-module']);
+        await resetExtension();
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(1, {
+            mock: 1
+        });
+        expect(zigbeeHerdsmanConverters.addDeviceDefinition).toHaveBeenNthCalledWith(2, {
+            mock: 2
+        });
+    });
+});

test/externalExtension.test.js

@@ -0,0 +1,113 @@
+const data = require('./stub/data');
+const logger = require('./stub/logger');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const path = require('path');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = require('./lib/flushPromises');
+const mocksClear = [
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
+    zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
+];
+
+const fs = require('fs');
+const mkdirSyncSpy = jest.spyOn(fs, 'mkdirSync');
+const unlinkSyncSpy = jest.spyOn(fs, 'unlinkSync');
+
+describe('User extensions', () => {
+    let controller;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(() => {
+        zigbeeHerdsman.returnDevices.splice(0);
+        controller = new Controller(jest.fn(), jest.fn());
+        mocksClear.forEach((m) => m.mockClear());
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        data.writeDefaultState();
+    });
+    afterEach(() => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        fs.rmdirSync(extensionPath, {recursive: true});
+    })
+
+    it('Load user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+    });
+
+    it('Load user extension from api call', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "foo.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'call from constructor', { retain: false, qos: 0 }, expect.any(Function));
+        expect(mkdirSyncSpy).toHaveBeenCalledWith(extensionPath);
+    });
+
+    it('Do not load corrupted extensions', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = "definetly not a correct javascript code";
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/save', stringify({"name": "foo.js", "code": extensionCode}));
+        await flushPromises();
+
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/response/extension/save', stringify({"data":{},"error":"Unexpected identifier","status":"error"}), { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Removes user extension', async () => {
+        const extensionPath = path.join(data.mockDir, 'extension');
+        const extensionCode = fs.readFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), 'utf-8');
+        fs.mkdirSync(extensionPath);
+        const extensionFilePath = path.join(extensionPath, 'exampleExtension.js')
+        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), extensionFilePath)
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/extensions', stringify([{"name": "exampleExtension.js", "code": extensionCode}]), { retain: true, qos: 0 }, expect.any(Function));
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "exampleExtension.js"}));
+        await flushPromises();
+        expect(unlinkSyncSpy).toHaveBeenCalledWith(extensionFilePath);
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/extension/remove', stringify({"name": "non existing.js"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/extension/remove',
+            stringify({"data":{},"status":"error","error":"Extension non existing.js doesn't exists"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+});

test/frontend.test.js

@@ -0,0 +1,224 @@
+const data = require('./stub/data');
+require('./stub/logger');
+require('./stub/zigbeeHerdsman');
+const MQTT = require('./stub/mqtt');
+const settings = require('../lib/util/settings');
+const Controller = require('../lib/controller');
+const stringify = require('json-stable-stringify-without-jsonify');
+const flushPromises = require('./lib/flushPromises');
+const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+jest.spyOn(process, 'exit').mockImplementation(() => {});
+
+const mockHTTP = {
+    implementation: {
+        listen: jest.fn(),
+        on: (event, handler) => {mockHTTP.events[event] = handler},
+        close: jest.fn().mockImplementation((cb) => cb()),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockWSocket = {
+    close: jest.fn(),
+};
+
+const mockWS = {
+    implementation: {
+        clients: [],
+        on: (event, handler) => {mockWS.events[event] = handler},
+        handleUpgrade: jest.fn().mockImplementation((request, socket, head, cb) => {
+            cb(mockWSocket)
+        }),
+        emit: jest.fn(),
+        close: jest.fn(),
+    },
+    variables: {},
+    events: {},
+};
+
+const mockNodeStatic = {
+    implementation: jest.fn(),
+    variables: {},
+    events: {},
+};
+
+jest.mock('http', () => ({
+    createServer: jest.fn().mockImplementation((onRequest) => {
+        mockHTTP.variables.onRequest = onRequest;
+        return mockHTTP.implementation;
+    }),
+}));
+
+jest.mock("serve-static", () =>
+    jest.fn().mockImplementation((path) => {
+        mockNodeStatic.variables.path = path
+        return mockNodeStatic.implementation
+    })
+);
+
+jest.mock('zigbee2mqtt-frontend', () => ({
+    getPath: () => 'my/dummy/path',
+}));
+
+jest.mock('ws', () => ({
+    OPEN: 'open',
+    Server: jest.fn().mockImplementation(() => {
+        return mockWS.implementation;
+    }),
+}));
+
+describe('Frontend', () => {
+    let controller;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
+
+    beforeEach(async () => {
+        mockWS.implementation.clients = [];
+        data.writeDefaultConfiguration();
+        data.writeDefaultState();
+        settings.reRead();
+        settings.set(['frontend'], {port: 8081, host: "127.0.0.1"});
+        settings.set(['homeassistant'], true);
+        zigbeeHerdsman.devices.bulb.linkquality = 10;
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    afterEach(async() => {
+        delete zigbeeHerdsman.devices.bulb.linkquality;
+    });
+
+    it('Start/stop', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        expect(mockNodeStatic.variables.path).toBe("my/dummy/path");
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+
+        const mockWSClient = {
+            implementation: {
+                terminate: jest.fn(),
+                send: jest.fn(),
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await controller.stop();
+        expect(mockWSClient.implementation.terminate).toHaveBeenCalledTimes(1);
+        expect(mockHTTP.implementation.close).toHaveBeenCalledTimes(1);
+        expect(mockWS.implementation.close).toHaveBeenCalledTimes(1);
+    });
+
+    it('Websocket interaction', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        // Connect
+        const mockWSClient = {
+            implementation: {
+                on: (event, handler) => {mockWSClient.events[event] = handler},
+                send: jest.fn(),
+                readyState: 'open',
+            },
+            events: {},
+        };
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bridge/state', payload: 'online'}));
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255}}));
+
+        // Message
+        MQTT.publish.mockClear();
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb_color',
+            stringify({state: 'ON', linkquality: null}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        mockWSClient.events.message(undefined, false);
+        mockWSClient.events.message("", false);
+        mockWSClient.events.message(null, false);
+        await flushPromises();
+
+        // Received message on socket
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(1);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic: 'bulb_color', payload: {state: 'ON', linkquality: null}}));
+
+        // Shouldnt set when not ready
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'close';
+        mockWSClient.events.message(stringify({topic: 'bulb_color/set', payload: {state: 'ON'}}), false)
+        expect(mockWSClient.implementation.send).toHaveBeenCalledTimes(0);
+
+        // Send last seen on connect
+        mockWSClient.implementation.send.mockClear();
+        mockWSClient.implementation.readyState = 'open';
+        settings.set(['advanced'], {last_seen: 'ISO_8601'});
+        mockWS.implementation.clients.push(mockWSClient.implementation);
+        await mockWS.events.connection(mockWSClient.implementation);
+        expect(mockWSClient.implementation.send).toHaveBeenCalledWith(stringify({topic:"remote", payload:{brightness:255, last_seen: "1970-01-01T00:00:01.000Z"}}));
+    });
+
+    it('onReques/onUpgrade', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: 'http://localhost:8080/api'}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "http://localhost:8080/api"}, mockSocket, 3, expect.any(Function));
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](99);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', 99, {"url": "http://localhost:8080/api"});
+
+        mockHTTP.variables.onRequest(1, 2);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledTimes(1);
+        expect(mockNodeStatic.implementation).toHaveBeenCalledWith(1, 2, expect.any(Function));
+    });
+
+    it('Static server', async () => {
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        expect(mockHTTP.implementation.listen).toHaveBeenCalledWith(8081, "127.0.0.1");
+    });
+
+    it('Authentification', async () => {
+        const authToken = 'sample-secure-token'
+        settings.set(['frontend'], {auth_token: authToken});
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+
+        const mockSocket = {destroy: jest.fn()};
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: '/api'}, mockSocket, mockWSocket);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({"url": "/api"}, mockSocket, mockWSocket, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledWith(4401, "Unauthorized");
+
+        mockWSocket.close.mockClear();
+        mockWS.implementation.emit.mockClear();
+
+        const url = `/api?token=${authToken}`;
+        mockWS.implementation.handleUpgrade.mockClear();
+        mockHTTP.events.upgrade({url: url}, mockSocket, 3);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledTimes(1);
+        expect(mockSocket.destroy).toHaveBeenCalledTimes(0);
+        expect(mockWS.implementation.handleUpgrade).toHaveBeenCalledWith({url}, mockSocket, 3, expect.any(Function));
+        expect(mockWSocket.close).toHaveBeenCalledTimes(0);
+        mockWS.implementation.handleUpgrade.mock.calls[0][3](mockWSocket);
+        expect(mockWS.implementation.emit).toHaveBeenCalledWith('connection', mockWSocket, {url});
+
+    });
+});

test/group.test.js

@@ -1,34 +1,54 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45542');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
+zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45724');
+
 const MQTT = require('./stub/mqtt');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const settings = require('../lib/util/settings');
 
 describe('Groups', () => {
     let controller;
 
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Groups');
+        await controller.enableDisableExtension(true, 'Groups');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
+        await controller.start();
+        await flushPromises();
+    });
+    
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     beforeEach(() => {
-        data.writeEmptyState();
-        controller = new Controller();
         Object.values(zigbeeHerdsman.groups).forEach((g) => g.members = []);
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         MQTT.publish.mockClear();
+        zigbeeHerdsman.groups.gledopto_group.command.mockClear();
         zigbeeHerdsmanConverters.toZigbeeConverters.__clearStore__();
+        controller.state.state = {};
     })
 
     it('Apply group updates add', async () => {
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb', 'bulb_color']}});
         zigbeeHerdsman.groups.group_1.members.push(zigbeeHerdsman.devices.bulb.getEndpoint(1))
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
             zigbeeHerdsman.devices.bulb.getEndpoint(1),
             zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
@@ -40,103 +60,110 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
+    it('Apply group updates remove handle fail', async () => {
+        const endpoint = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+        endpoint.removeFromGroup.mockImplementationOnce(() => {throw new Error("failed!")});
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false,}});
+        logger.error.mockClear();
+        await resetExtension();
+        expect(logger.error).toHaveBeenCalledWith(`Failed to remove 'bulb_color' from 'group_1'`);
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([endpoint]);
+    });
+
     it('Move to non existing group', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'3': {friendly_name: 'group_3', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
     });
 
     it('Add non standard endpoint to group with name', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['0x0017880104e45542/right']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(3)]);
     });
 
     it('Add non standard endpoint to group with number', async () => {
         const QBKG03LM = zigbeeHerdsman.devices.QBKG03LM;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['wall_switch_double/2']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([QBKG03LM.getEndpoint(2)]);
     });
 
     it('Shouldnt crash on non-existing devices', async () => {
         logger.error.mockClear();
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['not_existing_bla']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([]);
         expect(logger.error).toHaveBeenCalledWith("Cannot find 'not_existing_bla' of group 'group_1'");
     });
 
-    it('Add to group via MQTT', async () => {
+    it('Should resolve device friendly names', async () => {
+        settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'friendly_name'], 'bulb_friendly_name');
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['bulb_friendly_name', 'bulb_color']}});
+        await resetExtension();
+        expect(zigbeeHerdsman.groups.group_1.members).toStrictEqual([
+            zigbeeHerdsman.devices.bulb.getEndpoint(1),
+            zigbeeHerdsman.devices.bulb_color.getEndpoint(1)
+        ]);
+    });
+
+    it('Legacy api: Add to group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Add to group with slashes via MQTT', async () => {
+    it('Legacy api: Add to group with slashes via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups["group/with/slashes"];
         settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         MQTT.events.message('zigbee2mqtt/bridge/group/group/with/slashes/add', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}', {"retain": false, qos: 0}, expect.any(Function));
-
-        // Test if subscribed to topics with slashes
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/remove');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/group/+/+/+/+/+/add');
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_add","message":{"friendly_name":"bulb_color","group":"group/with/slashes"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Add to group via MQTT with postfix', async () => {
+    it('Legacy api: Add to group via MQTT with postfix', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         expect(group.members).toStrictEqual([endpoint]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+    it('Legacy api: Add to group via MQTT with postfix shouldnt add it twice', async () => {
         const device = zigbeeHerdsman.devices.QBKG03LM;
         const endpoint = device.getEndpoint(3);
         const group = zigbeeHerdsman.groups.group_1;
         expect(group.members.length).toBe(0);
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'wall_switch_double/right');
         await flushPromises();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', '0x0017880104e45542/3');
@@ -145,90 +172,84 @@ describe('Groups', () => {
         expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
     });
 
-    it('Remove from group via MQTT', async () => {
+    it('Legacy api: Remove from group via MQTT', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove","message":{"friendly_name":"bulb_color","group":"group_1"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
-    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+    it('Legacy api: Remove from group via MQTT when in zigbee but not in settings', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'bulb_color');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
     });
 
-    it('Remove from group via MQTT with postfix variant 1', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 1', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 2', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 2', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', 'wall_switch_double/3');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group via MQTT with postfix variant 3', async () => {
+    it('Legacy api: Remove from group via MQTT with postfix variant 3', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Remove from group all', async () => {
+    it('Legacy api: Remove from group all', async () => {
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", '{"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bridge/log", stringify({"type":"device_group_remove_all","message":{"friendly_name":"wall_switch_double"}}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Remove from group all deprecated', async () => {
@@ -237,26 +258,23 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
         await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/remove_all', '0x0017880104e45542/right');
         await flushPromises();
         expect(group.members).toStrictEqual([]);
         expect(settings.getGroup('group_1').devices).toStrictEqual([]);
     });
 
-    it('Log when adding to non-existing group', async () => {
-        await controller.start();
-        await flushPromises();
+    it('Legacy api: Log when adding to non-existing group', async () => {
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1_not_existing/add', 'bulb_color');
         await flushPromises();
         expect(logger.error).toHaveBeenCalledWith("Group 'group_1_not_existing' does not exist");
     });
 
-    it('Log when adding to non-existing device', async () => {
-        await controller.start();
-        await flushPromises();
+    it('Legacy api: Log when adding to non-existing device', async () => {
+        await resetExtension();
         logger.error.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb_color_not_existing');
         await flushPromises();
@@ -269,8 +287,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -278,8 +295,27 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should not republish identical optimistic group states', async () => {
+        const device1 = zigbeeHerdsman.devices.bulb_2;
+        const device2 = zigbeeHerdsman.devices.bulb_color_2;
+        const group = zigbeeHerdsman.groups.group_tradfri_remote;
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device1, endpoint: device1.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await zigbeeHerdsman.events.message({data: {onOff: 1}, cluster: 'genOnOff', device: device2, endpoint: device2.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(6);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_tradfri_remote", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_with_tradfri", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/ha_discovery_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/switch_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of all members when a group changes its state', async () => {
@@ -288,15 +324,72 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state change for group when members state change', async () => {
+        // Created for https://github.com/Koenkk/zigbee2mqtt/issues/5725
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'OFF'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should publish state of device with endpoint name', async () => {
+        const group = zigbeeHerdsman.groups.gledopto_group;
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/gledopto_group/set', stringify({state: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/GLEDOPTO_2ID", stringify({"state_cct":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/gledopto_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(group.command).toHaveBeenCalledTimes(1);
+        expect(group.command).toHaveBeenCalledWith("genOnOff", "on", {}, {});
+    });
+
+    it('Should publish state of group when specific state of specific endpoint is changed', async () => {
+        const group = zigbeeHerdsman.groups.gledopto_group;
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/GLEDOPTO_2ID/set', stringify({state_cct: 'ON'}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/GLEDOPTO_2ID", stringify({"state_cct":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/gledopto_group", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(group.command).toHaveBeenCalledTimes(0);
     });
 
     it('Should publish state change of all members when a group changes its state, filtered', async () => {
@@ -305,15 +398,14 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, filtered_attributes: ['brightness'], devices: [device.ieeeAddr]}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON', brightness: 100}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON', brightness: 100}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","brightness":100}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON","brightness":100}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Shouldnt publish group state change when a group is not optimistic', async () => {
@@ -322,8 +414,7 @@ describe('Groups', () => {
         const group = zigbeeHerdsman.groups.group_1;
         group.members.push(endpoint);
         settings.set(['groups'], {'1': {friendly_name: 'group_1', devices: [device.ieeeAddr], optimistic: false, retain: false}});
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
         const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint, type: 'attributeReport', linkquality: 10};
@@ -331,7 +422,7 @@ describe('Groups', () => {
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON","linkquality":10}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change of another group with shared device when a group changes its state', async () => {
@@ -344,16 +435,15 @@ describe('Groups', () => {
             '2': {friendly_name: 'group_2', retain: false, devices: [device.ieeeAddr]},
             '3': {friendly_name: 'group_3', retain: false, devices: []}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
         MQTT.publish.mockClear();
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"ON"}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via device', async () => {
@@ -367,17 +457,16 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({state:"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should not publish state change off if any lights within are still on when changed via shared group', async () => {
@@ -392,18 +481,17 @@ describe('Groups', () => {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false},
             '2': {friendly_name: 'group_2', retain: false, devices: [device_1.ieeeAddr]},
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/group_2/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/group_2/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_2", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off if all lights within turn off', async () => {
@@ -417,20 +505,46 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'OFF'}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", '{"state":"OFF","brightness":0}', {"retain": true, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"OFF"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should only update group state with changed properties', async () => {
+        const device_1 = zigbeeHerdsman.devices.bulb_color;
+        const device_2 = zigbeeHerdsman.devices.bulb;
+        const endpoint_1 = device_1.getEndpoint(1);
+        const endpoint_2 = device_2.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint_1);
+        group.members.push(endpoint_2);
+        settings.set(['groups'], {
+            '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
+        });
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF', color_temp: 200}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON', color_temp: 250}));
+        await flushPromises();
+        MQTT.publish.mockClear();
+
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 300}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode": "color_temp","color_temp":300,"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode": "color_temp","color_temp":300,"state":"ON"}), {"retain": false, qos: 0}, expect.any(Function));
     });
 
     it('Should publish state change off even when missing current state', async () => {
@@ -444,19 +558,314 @@ describe('Groups', () => {
         settings.set(['groups'], {
             '1': {friendly_name: 'group_1', devices: [device_1.ieeeAddr, device_2.ieeeAddr], retain: false}
         });
-        await controller.start();
-        await flushPromises();
+        await resetExtension();
 
-        await MQTT.events.message('zigbee2mqtt/group_1/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({state: 'ON'}));
         await flushPromises();
         MQTT.publish.mockClear();
         controller.state.state = {};
 
-        await MQTT.events.message('zigbee2mqtt/bulb_color/set', JSON.stringify({state: 'OFF'}));
+        await MQTT.events.message('zigbee2mqtt/bulb_color/set', stringify({state: 'OFF'}));
         await flushPromises();
 
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
-        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", '{"state":"OFF","brightness":0}', {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"state":"OFF"}), {"retain": false, qos: 0}, expect.any(Function));
     });
-});
+
+    it('Add to group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({transaction: "123", group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"transaction": "123", "status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await resetExtension();
+        endpoint.addToGroup.mockImplementationOnce(() => {throw new Error('timeout')});
+        await flushPromises();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"error","error":"Failed to add from group (timeout)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group with slashes via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups["group/with/slashes"];
+        settings.set(['groups'], {'99': {friendly_name: 'group/with/slashes', retain: false, devices: []}});
+        expect(group.members.length).toBe(0);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group/with/slashes', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group/with/slashes').devices).toStrictEqual([`${device.ieeeAddr}/1`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color","group":"group/with/slashes"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Add to group via MQTT with postfix shouldnt add it twice', async () => {
+        const device = zigbeeHerdsman.devices.QBKG03LM;
+        const endpoint = device.getEndpoint(3);
+        const group = zigbeeHerdsman.groups.group_1;
+        expect(group.members.length).toBe(0);
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'wall_switch_double/right'}));
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([endpoint]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([`${device.ieeeAddr}/${endpoint.ID}`]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"wall_switch_double/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT keeping device reporting', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [device.ieeeAddr]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT when in zigbee but not in settings', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: ['dummy']}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'bulb_color'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual(['dummy']);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 1', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/right`]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 2', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`0x0017880104e45542/right`]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: 'wall_switch_double/3'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"wall_switch_double/3","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group via MQTT with postfix variant 3', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1', device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"0x0017880104e45542/right","group":"group_1"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Remove from group all', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(endpoint);
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [`wall_switch_double/3`]}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove_all', stringify({device: '0x0017880104e45542/right'}));
+        await flushPromises();
+        expect(group.members).toStrictEqual([]);
+        expect(settings.getGroup('group_1').devices).toStrictEqual([]);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove_all',
+            stringify({"data":{"device":"0x0017880104e45542/right"},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing group', async () => {
+        await resetExtension();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/remove', stringify({group: 'group_1_not_existing', device: 'bulb_color'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/remove',
+            stringify({"data":{"device":"bulb_color","group":"group_1_not_existing"},"status":"error","error":"Group 'group_1_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Error when adding to non-existing device', async () => {
+        await resetExtension();
+        logger.error.mockClear();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/group/members/add', stringify({group: 'group_1', device: 'bulb_color_not_existing'}));
+        await flushPromises();
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bridge/groups', expect.any(String), expect.any(Object), expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/group/members/add',
+            stringify({"data":{"device":"bulb_color_not_existing","group":"group_1"},"status":"error","error":"Device 'bulb_color_not_existing' does not exist"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should only include relevant properties when publishing member states', async () => {
+        const bulbColor = zigbeeHerdsman.devices.bulb_color;
+        const bulbColorTemp = zigbeeHerdsman.devices.bulb;
+        const group = zigbeeHerdsman.groups.group_1;
+        group.members.push(bulbColor.getEndpoint(1));
+        group.members.push(bulbColorTemp.getEndpoint(1));
+        settings.set(['groups'], {'1': {friendly_name: 'group_1', retain: false, devices: [bulbColor.ieeeAddr, bulbColorTemp.ieeeAddr]}});
+        await resetExtension();
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color_temp: 50}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":50}), {"retain": true, qos: 0}, expect.any(Function));
+
+        MQTT.publish.mockClear();
+        await MQTT.events.message('zigbee2mqtt/group_1/set', stringify({color: {x: 0.5, y: 0.3}}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb_color", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/group_1", stringify({"color":{"x":0.5,"y":0.3},"color_mode":"xy","color_temp":548}), {"retain": false, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"color_mode":"color_temp","color_temp":548}), {"retain": true, qos: 0}, expect.any(Function));
+    });
+});

test/legacy/bridgeLegacy.test.js

@@ -2,34 +2,40 @@ const data = require('../stub/data');
 const logger = require('../stub/logger');
 const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 const MQTT = require('../stub/mqtt');
+const stringify = require('json-stable-stringify-without-jsonify');
 const path = require('path');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../../lib/util/settings');
 const Controller = require('../../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('../lib/flushPromises');
 
 
 describe('Bridge legacy', () => {
     let controller;
+    let version;
 
     beforeAll(async () => {
-        this.version = await require('../../lib/util/utils').getZigbee2mqttVersion();
-        controller = new Controller();
+        jest.useFakeTimers();
+        version = await require('../../lib/util/utils').getZigbee2MQTTVersion();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
     })
 
     beforeEach(() => {
         data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeDefaultState();
+        settings.reRead();
         logger.info.mockClear();
         logger.warn.mockClear();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
     it('Should publish bridge configuration on startup', async () => {
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/config',
-          JSON.stringify({"version":this.version.version,"commit":this.version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"log_level":'info',"permit_join":false}),
+          stringify({"version":version.version,"commit":version.commitHash,"coordinator":{"type":"z-Stack","meta":{"version":1, "revision": 20190425}},"network":{"panID":5674,"extendedPanID":[0,11,22],"channel":15},"log_level":'info',"permit_join":false}),
           { retain: true, qos: 0 },
           expect.any(Function)
         );
@@ -56,7 +62,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb_color"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -67,7 +73,7 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-          JSON.stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
+          stringify({type: "device_whitelisted", "message": {friendly_name: "bulb"}}),
           { retain: false, qos: 0 },
           expect.any(Function)
         );
@@ -85,12 +91,12 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', optionswrong: {retain: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
@@ -100,17 +106,17 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {random_setting: true}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_1: true}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true}}
         );
-        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', JSON.stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/device_options', stringify({friendly_name: 'bulb_color', options: {options: {random_2: false}}}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(
             {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "random_setting": true, "retain": true, options: {random_1: true, random_2: false}}
@@ -122,12 +128,12 @@ describe('Bridge legacy', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'true');
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
         zigbeeHerdsman.permitJoin.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/permit_join', 'false');
         await flushPromises();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
     });
 
     it('Should allow to reset', async () => {
@@ -177,7 +183,7 @@ describe('Bridge legacy', () => {
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
         expect(payload.length).toStrictEqual(Object.values(zigbeeHerdsman.devices).length);
         expect(payload[1]).toStrictEqual({"ieeeAddr": "0x00124b00120144ae", "type": "Coordinator", "dateCode": "20190425", "friendly_name": "Coordinator", networkAddress: 0, softwareBuildID: "z-Stack", lastSeen: 100});
-        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
+        expect(payload[2]).toStrictEqual({"dateCode": null, "friendly_name": "bulb", "ieeeAddr": "0x000b57fffec6a5b2", "lastSeen": 1000, "manufacturerID": 4476, "model": "LED1545G12", "modelID": "TRADFRI bulb E27 WS opal 980lm", "networkAddress": 40369, "powerSource": "Mains (single phase)", "softwareBuildID": null, "type": "Router", "description": "TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white", "vendor": "IKEA"});
         Date.now = now;
     });
 
@@ -187,25 +193,25 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         const payload = JSON.parse(MQTT.publish.mock.calls[0][1]);
-        expect(payload).toStrictEqual({"message": [{"ID": 1, "friendly_name": "group_1", "retain": false, 'devices': [], optimistic: true}, {"ID": 2, "friendly_name": "group_2", "retain": false, "devices": [], optimistic: true}, {"ID": 11, "friendly_name": "group_with_tradfri", "retain": false, "devices": ['bulb_2'], optimistic: true}, {"ID": 15071, "friendly_name": "group_tradfri_remote", "retain": false, "devices": ['bulb_color_2', 'bulb_2'], optimistic: true}], "type": "groups"});
+        expect(payload).toStrictEqual({"message":[{"ID":1,"devices":[],"friendly_name":"group_1","retain":false},{"ID":2,"devices":[],"friendly_name":"group_2","retain":false},{"ID":9,"devices":["bulb_color_2","bulb_2","wall_switch_double/right"],"friendly_name":"ha_discovery_group"},{"ID":11,"devices":["bulb_2"],"friendly_name":"group_with_tradfri","retain":false},{"ID":12,"devices":["TS0601_thermostat"],"friendly_name":"thermostat_group","retain":false},{"ID":14,"devices":["power_plug","bulb_2"],"friendly_name":"switch_group","retain":false},{"ID":21,"devices":["GLEDOPTO_2ID/cct"],"friendly_name":"gledopto_group"},{"ID":15071,"devices":["bulb_color_2","bulb_2"],"friendly_name":"group_tradfri_remote","retain":false}],"type":"groups"});
     });
 
     it('Should allow rename devices', async () => {
         const bulb_color2 = {"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color2", "friendly_name": "bulb_color2", "retain": false};
         MQTT.publish.mockClear();
         expect(settings.getDevice('bulb_color')).toStrictEqual({"ID": "0x000b57fffec6a5b3", "friendlyName": "bulb_color", "friendly_name": "bulb_color", "retain": false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color2'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color2'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color')).toStrictEqual(null);
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb_color', to: 'bulb_color2'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color2', newmalformed: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
@@ -213,20 +219,20 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
 
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'bulb_color', new: 'bulb_color3'}));
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'bulb_color', new: 'bulb_color3'}));
         await flushPromises();
         expect(settings.getDevice('bulb_color2')).toStrictEqual(bulb_color2);
     });
 
     it('Should allow rename groups', async () => {
         MQTT.publish.mockClear();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", optimistic: true, retain: false});
-        MQTT.events.message('zigbee2mqtt/bridge/config/rename', JSON.stringify({old: 'group_1', new: 'group_1_renamed'}));
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1", "friendly_name": "group_1", retain: false});
+        MQTT.events.message('zigbee2mqtt/bridge/config/rename', stringify({old: 'group_1', new: 'group_1_renamed'}));
         await flushPromises();
-        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", optimistic: true, retain: false});
+        expect(settings.getGroup(1)).toStrictEqual({"ID": 1, devices: [], friendlyName: "group_1_renamed", "friendly_name": "group_1_renamed", retain: false});
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
+            stringify({type: 'group_renamed', message: {from: 'group_1', to: 'group_1_renamed'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -243,14 +249,14 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb_new_name');
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
+            stringify({type: 'device_renamed', message: {from: 'bulb', to: 'bulb_new_name'}}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
     });
 
     it('Shouldnt rename when no device has been joined', async () => {
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
         expect(settings.getDevice('0x000b57fffec6a5b2').friendlyName).toStrictEqual('bulb');
@@ -265,11 +271,11 @@ describe('Bridge legacy', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
     });
@@ -278,12 +284,12 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group"}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 3, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(3);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -293,12 +299,12 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"friendly_name": "new_group", "id": 42}');
         await flushPromises();
-        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: [], optimistic: true});
+        expect(settings.getGroup('new_group')).toStrictEqual({"ID": 42, "friendlyName": "new_group", "friendly_name": "new_group", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_added', message: 'new_group'}),
+            stringify({type: 'group_added', message: 'new_group'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -308,7 +314,7 @@ describe('Bridge legacy', () => {
         zigbeeHerdsman.createGroup.mockClear();
         MQTT.events.message('zigbee2mqtt/bridge/config/add_group', '{"id": 42}');
         await flushPromises();
-        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: [], optimistic: true});
+        expect(settings.getGroup('group_42')).toStrictEqual({"ID": 42, "friendlyName": "group_42", "friendly_name": "group_42", devices: []});
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.createGroup).toHaveBeenCalledWith(42)
     });
@@ -318,10 +324,24 @@ describe('Bridge legacy', () => {
         MQTT.events.message('zigbee2mqtt/bridge/config/remove_group', 'group_1');
         await flushPromises();
         expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
+        expect(group.removeFromNetwork).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/log',
+            stringify({type: 'group_removed', message: 'group_1'}),
+            {qos: 0, retain: false},
+            expect.any(Function)
+        );
+    });
+
+    it('Should allow to force remove groups', async () => {
+        const group = zigbeeHerdsman.groups.group_1;
+        MQTT.events.message('zigbee2mqtt/bridge/config/force_remove_group', 'group_1');
+        await flushPromises();
+        expect(settings.getGroup('to_be_removed')).toStrictEqual(null);
         expect(group.removeFromDatabase).toHaveBeenCalledTimes(1);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'group_removed', message: 'group_1'}),
+            stringify({type: 'group_removed', message: 'group_1'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -356,7 +376,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_removed', message: 'bulb_color'}),
+            stringify({type: 'device_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -378,7 +398,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_force_removed', message: 'bulb_color'}),
+            stringify({type: 'device_force_removed', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -399,7 +419,7 @@ describe('Bridge legacy', () => {
         expect(settings.getDevice('bulb_color')).toBeNull();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/log',
-            JSON.stringify({type: 'device_banned', message: 'bulb_color'}),
+            stringify({type: 'device_banned', message: 'bulb_color'}),
             {qos: 0, retain: false},
             expect.any(Function)
         );
@@ -442,22 +462,22 @@ describe('Bridge legacy', () => {
     });
 
     it('Should allow to touchlink factory reset (OK)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(true);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(true);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.info).toHaveBeenCalledWith('Successfully factory reset device through Touchlink');
     });
 
     it('Should allow to touchlink factory reset (FAILS)', async () => {
-        zigbeeHerdsman.touchlinkFactoryReset.mockClear();
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockClear();
 
-        zigbeeHerdsman.touchlinkFactoryReset.mockReturnValueOnce(false);
+        zigbeeHerdsman.touchlinkFactoryResetFirst.mockReturnValueOnce(false);
         MQTT.events.message('zigbee2mqtt/bridge/config/touchlink/factory_reset', '');
         await flushPromises();
-        expect(zigbeeHerdsman.touchlinkFactoryReset).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.touchlinkFactoryResetFirst).toHaveBeenCalledTimes(1);
         expect(logger.warn).toHaveBeenCalledWith('Failed to factory reset device through Touchlink');
     });
 });

test/legacy/report.test.js

@@ -1,6 +1,6 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
+const data = require('../stub/data');
+const logger = require('../stub/logger');
+const zigbeeHerdsman = require('../stub/zigbeeHerdsman');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
 zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b2');
@@ -8,22 +8,25 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
 zigbeeHerdsman.returnDevices.push('0x0017880104e45559');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b4');
 zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b7');
+zigbeeHerdsman.returnDevices.push('0x0017880104e45524');
 zigbeeHerdsman.returnDevices.push('0x90fd9ffffe4b64ax');
-const MQTT = require('./stub/mqtt');
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const MQTT = require('../stub/mqtt');
+const settings = require('../../lib/util/settings');
+const Controller = require('../../lib/controller');
+const flushPromises = require('../lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-jest.mock('debounce', () => jest.fn(fn => fn));
-const debounce = require('debounce');
 
-const mocksClear = [MQTT.publish, logger.warn, logger.debug, debounce];
+const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Report', () => {
     let controller;
+    let extension;
 
     function expectOnOffBrightnessColorReport(endpoint, colorXY) {
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(1);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(0);
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
         expect(endpoint.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
         expect(endpoint.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
@@ -38,32 +41,89 @@ describe('Report', () => {
         }
     }
 
-    mockClear = (device) => {
+    function expectOnOffBrightnessColorReportDisabled(endpoint, colorXY) {
+        const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        const device = endpoint.getDevice();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
+        expect(endpoint.unbind).toHaveBeenCalledWith('lightingColorCtrl', coordinatorEndpoint);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(endpoint.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        if (colorXY) {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentX", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}, {"attribute": "currentY", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        } else {
+            expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 3, "reportableChange": 1}]);
+        }
+    }
+
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
             endpoint.configureReporting.mockClear();
             endpoint.bind.mockClear();
+            endpoint.unbind.mockClear();
         }
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
+    beforeAll(async () => {
+        jest.useFakeTimers();
         settings.set(['advanced', 'report'], true);
-        controller = new Controller();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        extension = controller.extensions.find((e) => e.constructor.name === 'Report');
+    });
+
+    beforeEach(async () => {
+        extension.enabled = true;
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        for (const device of Object.values(zigbeeHerdsman.devices)) {
+            mockClear(device);
+            delete device.meta.reporting;
+        }
         mocksClear.forEach((m) => m.mockClear());
-        await flushPromises();
+        extension.queue = new Set();
+        extension.failed = new Set();
+        await extension.onZigbeeStarted();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should configure reporting on startup', async () => {
+        await extension.onZigbeeStarted();
         const device = zigbeeHerdsman.devices.bulb_color;
         const endpoint = device.getEndpoint(1);
         expectOnOffBrightnessColorReport(endpoint, true);
     });
 
+    it('Should not configure reporting on startup when disabled', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        delete device.meta.report;
+        extension.enabled = false;
+        await extension.onZigbeeStarted();
+        expect(device.meta.reporting).toBe(undefined);
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should disable reporting on startup when enabled earlier', async () => {
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.meta.reporting = 1;
+        const endpoint = device.getEndpoint(1);
+        extension.enabled = false;
+        mockClear(device);
+        await extension.onZigbeeStarted();
+        expectOnOffBrightnessColorReportDisabled(endpoint, true);
+    });
+
     it('Should configure reporting when receicing message from device which has not been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
@@ -79,7 +139,6 @@ describe('Report', () => {
     });
 
     it('Should not configure reporting when still configuring', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(1000));
@@ -91,8 +150,6 @@ describe('Report', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        jest.runAllTimers();
-        jest.useRealTimers();
     });
 
     it('Should not mark as configured when reporting setup fails', async () => {
@@ -108,6 +165,20 @@ describe('Report', () => {
         expect(device.meta.reporting).toBeUndefined();
     });
 
+    it('Should not configure reporting when interviewing', async () => {
+        const device = zigbeeHerdsman.devices.bulb_2;
+        const endpoint = device.getEndpoint(1);
+        device.interviewing = true;
+        delete device.meta.reporting;
+        mockClear(device);
+        const payload = {data: {onOff: 1}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expect(endpoint.configureReporting).toHaveBeenCalledTimes(0);
+        expect(device.meta.reporting).toBeUndefined();
+    });
+
     it('Should not configure reporting when receicing message from device which has already been setup yet', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
@@ -128,7 +199,7 @@ describe('Report', () => {
     });
 
     it('Should configure reporting when deviceAnnounce message from IKEA device', async () => {
-        const device = zigbeeHerdsman.devices.bulb;
+        const device = zigbeeHerdsman.devices.bulb_2;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         const payload = {device};
@@ -173,36 +244,6 @@ describe('Report', () => {
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
     });
 
-    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
-        const remote = zigbeeHerdsman.devices.remote;
-        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
-        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-    });
-
-    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote and should', async () => {
-        const remote = zigbeeHerdsman.devices.tradfri_remote;
-        const data = {"stepmode":0,"stepsize":43,"transtime":5};
-        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
-
-        // Should also only debounce once
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(debounce).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
-
-        // Should only call Hue bulb, not e.g. tradfri
-        expect(zigbeeHerdsman.devices.bulb.getEndpoint(1).read).toHaveBeenCalledTimes(0);
-    });
-
     it('Should not configure reporting for the ZNLDP12LM closuresWindowCovering as it is ignored', async () => {
         const device = zigbeeHerdsman.devices.ZNLDP12LM;
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
@@ -220,6 +261,8 @@ describe('Report', () => {
         const device = zigbeeHerdsman.devices.bulb;
         const coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+        const configuredReportings = endpoint.configuredReportings;
+        endpoint.configuredReportings = [];
         delete device.meta.reporting;
         mockClear(device);
         endpoint.getClusterAttributeValue = jest.fn();
@@ -241,5 +284,6 @@ describe('Report', () => {
         expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities'])
         expect(endpoint.configureReporting).toHaveBeenCalledWith('lightingColorCtrl', [{"attribute": "colorTemperature", "maximumReportInterval": 300, "minimumReportInterval": 3, "reportableChange": 1}]);
         expect(endpoint.configureReporting).toHaveBeenCalledTimes(3);
+        endpoint.configuredReportings = configuredReportings;
     });
 });

test/lib/flushPromises.js

@@ -0,0 +1,2 @@
+const globalSetImmediate = setImmediate;
+module.exports = () => new Promise(globalSetImmediate);

test/logger.test.js

@@ -6,6 +6,7 @@ const path = require('path');
 const data = require('./stub/data');
 let stdOutWriteOriginal;
 const rimraf = require('rimraf');
+const Transport = require('winston-transport');
 
 describe('Logger', () => {
     beforeEach(async () => {
@@ -13,7 +14,7 @@ describe('Logger', () => {
         jest.resetModules();
         settings = require('../lib/util/settings');
         settings.set(['advanced', 'log_directory'], dir.name + '/%TIMESTAMP%');
-        settings._reRead();
+        settings.reRead();
         stdOutWriteOriginal = console._stdout.write;
         console._stdout.write = () => {};
     });
@@ -24,12 +25,14 @@ describe('Logger', () => {
 
     it('Create log directory', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const dirs = fs.readdirSync(dir.name);
         expect(dirs.length).toBe(1);
     });
 
     it('Should cleanup', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
 
         for (const d of fs.readdirSync(dir.name)) {
             rimraf.sync(path.join(dir.name, d));
@@ -46,6 +49,7 @@ describe('Logger', () => {
 
     it('Should not cleanup when there is no timestamp set', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         for (let i = 30; i < 40; i++) {
             fs.mkdirSync(path.join(dir.name, `log_${i}`));
         }
@@ -58,12 +62,34 @@ describe('Logger', () => {
 
     it('Set and get log level', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         logger.setLevel('debug');
         expect(logger.getLevel()).toBe('debug');
     });
 
+    it('Add transport', () => {
+        class DummyTransport extends Transport {
+            log(info, callback) {
+            }
+        }
+
+        const logger = require('../lib/util/logger.js');
+        expect(logger.transports.length).toBe(2);
+        logger.addTransport(new DummyTransport());
+        expect(logger.transports.length).toBe(3);
+    });
+
+    it('Set and get log level warn <-> warning', () => {
+        const logger = require('../lib/util/logger.js');
+        logger.logOutput();
+        logger.setLevel('warn');
+        expect(logger.transports[0].level).toBe('warning');
+        expect(logger.getLevel()).toBe('warn');
+    });
+
     it('Logger should be console and file by default', () => {
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -75,6 +101,7 @@ describe('Logger', () => {
     it('Logger can be file only', () => {
         settings.set(['advanced', 'log_output'], ['file']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.length).toBe(2);
         expect(pipes[0].constructor.name).toBe('Console');
@@ -86,6 +113,7 @@ describe('Logger', () => {
     it('Logger can be console only', () => {
         settings.set(['advanced', 'log_output'], ['console']);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(false);
@@ -94,6 +122,7 @@ describe('Logger', () => {
     it('Logger can be nothing', () => {
         settings.set(['advanced', 'log_output'], []);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes.constructor.name).toBe('Console');
         expect(pipes.silent).toBe(true);
@@ -102,10 +131,21 @@ describe('Logger', () => {
     it('Should allow to disable log rotation', () => {
         settings.set(['advanced', 'log_rotation'], false);
         const logger = require('../lib/util/logger.js');
+        logger.logOutput();
         const pipes = logger._readableState.pipes;
         expect(pipes[1].constructor.name).toBe('File');
         expect(pipes[1].maxFiles).toBeNull();
         expect(pipes[1].tailable).toBeFalsy();
         expect(pipes[1].maxsize).toBeNull();
     });
+
+    it('Should allow to symlink logs to current directory', () => {
+        settings.set(['advanced', 'log_symlink_current'], true);
+        let logger = require('../lib/util/logger.js');
+        logger.logOutput();
+        expect(fs.readdirSync(dir.name).includes('current')).toBeTruthy()
+
+        jest.resetModules();
+        logger = require('../lib/util/logger.js');
+    });
 });

test/networkMap.test.js

@@ -1,32 +1,38 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-
-const {coordinator, bulb, bulb_color, WXKG02LM, CC2530_ROUTER, unsupported_router} = zigbeeHerdsman.devices;
+const stringify = require('json-stable-stringify-without-jsonify');
+const fs = require('fs');
+const path = require('path');
+const {coordinator, bulb, bulb_color, WXKG02LM_rev1, CC2530_ROUTER, unsupported_router, external_converter_device} = zigbeeHerdsman.devices;
 
 zigbeeHerdsman.returnDevices.push(coordinator.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(bulb.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(bulb_color.ieeeAddr);
-zigbeeHerdsman.returnDevices.push(WXKG02LM.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(WXKG02LM_rev1.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(CC2530_ROUTER.ieeeAddr);
 zigbeeHerdsman.returnDevices.push(unsupported_router.ieeeAddr);
+zigbeeHerdsman.returnDevices.push(external_converter_device.ieeeAddr)
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-Date.now = jest.fn()
-Date.now.mockReturnValue(10000);
+const flushPromises = require('./lib/flushPromises');
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const setTimeoutNative = setTimeout;
 
 describe('Networkmap', () => {
     let controller;
 
     beforeAll(async () => {
+        jest.useFakeTimers();
+        Date.now = jest.fn()
+        Date.now.mockReturnValue(10000);
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeEmptyState();
-        controller = new Controller();
+        fs.copyFileSync(path.join(__dirname, 'assets', 'mock-external-converter.js'), path.join(data.mockDir, 'mock-external-converter.js'));
+        settings.set(['external_converters'], ['mock-external-converter.js']);
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
@@ -35,22 +41,35 @@ describe('Networkmap', () => {
     beforeEach(async () => {
         mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = 1000;
+        external_converter_device.lastSeen = 1000;
+        global.setTimeout = (r) => r();
+    });
+
+    afterEach(async () => {
+        global.setTimeout = setTimeoutNative;
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     function mock() {
         /**
          * Topology
-         *
+         *             | -> external_device
          *             | -> bulb_color -> unsupported_router (offline)
          * coordinator |      ^     ^
          *             |      |     | (not valid)
          *             | -> bulb    |
-         *                    |  -> CC2530_ROUTER -> WXKG02LM
+         *                    |  -> CC2530_ROUTER -> WXKG02LM_rev1
          *
          */
         coordinator.lqi = () => {return {neighbors: [
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 2, depth: 1, linkquality: 120},
-            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92}
+            {ieeeAddr: bulb.ieeeAddr, networkAddress: bulb.networkAddress, relationship: 2, depth: 1, linkquality: 92},
+            {ieeeAddr: external_converter_device.ieeeAddr, networkAddress: external_converter_device.networkAddress, relationship: 2, depth: 1, linkquality: 92}
         ]}};
         coordinator.routingTable = () => {return {table: [
             {destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress},
@@ -66,7 +85,7 @@ describe('Networkmap', () => {
         bulb_color.routingTable = () => {return {table: []}};
 
         CC2530_ROUTER.lqi = () => {return {neighbors: [
-            {ieeeAddr: WXKG02LM.ieeeAddr, networkAddress: WXKG02LM.networkAddress, relationship: 1, depth: 2, linkquality: 130},
+            {ieeeAddr: '0x0000000000000000', networkAddress: WXKG02LM_rev1.networkAddress, relationship: 1, depth: 2, linkquality: 130},
             {ieeeAddr: bulb_color.ieeeAddr, networkAddress: bulb_color.networkAddress, relationship: 4, depth: 2, linkquality: 130},
         ]}};
         CC2530_ROUTER.routingTable = () => {return {table: []}};
@@ -75,7 +94,7 @@ describe('Networkmap', () => {
         unsupported_router.routingTable = () => {throw new Error('failed')};
     }
 
-    it('Output raw networkmap', async () => {
+    it('Output raw networkmap legacy api', async () => {
         mock();
         MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'raw');
         await flushPromises();
@@ -83,26 +102,7 @@ describe('Networkmap', () => {
         let call = MQTT.publish.mock.calls[0];
         expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/raw');
 
-        const conv = (device) => {return {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress}};
-
-        const expected = {
-            "nodes":[
-                {"lastSeen": 1000,"ieeeAddr":coordinator.ieeeAddr,"friendlyName":"Coordinator","type":"Coordinator","networkAddress":0,"modelID":null,"failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb.ieeeAddr,"friendlyName":"bulb","type":"Router","networkAddress":40369,"modelID":"TRADFRI bulb E27 WS opal 980lm","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":bulb_color.ieeeAddr,"friendlyName":"bulb_color","type":"Router","networkAddress":40399,"modelID":"LLC020","failed":[]},
-                {"lastSeen": 1000,"ieeeAddr":WXKG02LM.ieeeAddr,"friendlyName":"button_double_key","type":"EndDevice","networkAddress":6538,"modelID":"lumi.sensor_86sw2.es1"},
-                {"lastSeen": 1000,"ieeeAddr":unsupported_router.ieeeAddr,"friendlyName":"0x0017880104e45525","type":"Router","networkAddress":6536,"modelID":"notSupportedModelID","manufacturerName": "Boef","failed":['lqi', 'routingTable']},
-                {"lastSeen": 1000,"ieeeAddr":CC2530_ROUTER.ieeeAddr,"friendlyName":"cc2530_router","type":"Router","networkAddress":6540,"modelID":"lumi.router","failed":[]},
-            ],
-            "links":[
-                {depth: 1, linkquality: 120, routes: [], source: conv(bulb_color), target: conv(coordinator), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 120, relationship: 2},
-                {depth: 1, linkquality: 92, routes: [{destinationAddress: CC2530_ROUTER.networkAddress, status: 'ACTIVE', nextHop: bulb.networkAddress}], source: conv(bulb), target: conv(coordinator), sourceIeeeAddr: bulb.ieeeAddr, sourceNwkAddr: bulb.networkAddress, targetIeeeAddr: coordinator.ieeeAddr, lqi: 92, relationship: 2},
-                {depth: 2, linkquality: 110, routes: [], source: conv(bulb_color), target: conv(bulb), sourceIeeeAddr: bulb_color.ieeeAddr, sourceNwkAddr: bulb_color.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 110, relationship: 1},
-                {depth: 2, linkquality: 100, routes: [], source: conv(CC2530_ROUTER), target: conv(bulb), sourceIeeeAddr: CC2530_ROUTER.ieeeAddr, sourceNwkAddr: CC2530_ROUTER.networkAddress, targetIeeeAddr: bulb.ieeeAddr, lqi: 100, relationship: 1},
-                {depth: 2, linkquality: 130, routes: [], source: conv(WXKG02LM), target: conv(CC2530_ROUTER), sourceIeeeAddr: WXKG02LM.ieeeAddr, sourceNwkAddr: WXKG02LM.networkAddress, targetIeeeAddr: CC2530_ROUTER.ieeeAddr, lqi: 130, relationship: 1}
-            ]
-        };
-
+        const expected =  {"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description": "external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]};
         expect(JSON.parse(call[1])).toStrictEqual(expected);
 
         /**
@@ -126,11 +126,7 @@ describe('Networkmap', () => {
         expect(JSON.parse(call[1])).toStrictEqual(expected);
     });
 
-    it('Output graphviz networkmap', async () => {
-        const getTimezoneOffset = Date.prototype.getTimezoneOffset;
-        const getHours = Date.prototype.getHours;
-        Date.prototype.getTimezoneOffset = () => -60;
-        Date.prototype.getHours = () => 1;
+    it('Output graphviz networkmap legacy api', async () => {
         mock();
         const device = zigbeeHerdsman.devices.bulb_color;
         device.lastSeen = null;
@@ -146,17 +142,19 @@ describe('Networkmap', () => {
 
         const expected = `digraph G {
             node[shape=record];
-              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0)|1970-01-01T01:00:10+01:00}"];
-              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (40369)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|1970-01-01T01:00:01+01:00}"];
-              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 6540)"]
-              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (40399)|Philips Hue Go (7146060PH)|unknown}"];
+              "0x00124b00120144ae" [style="bold, filled", fillcolor="#e04e5d", fontcolor="#ffffff", label="{Coordinator|0x00124b00120144ae (0x0000)|0 seconds ago}"];
+              "0x000b57fffec6a5b2" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb|0x000b57fffec6a5b2 (0x9db1)|IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)|9 seconds ago}"];
+              "0x000b57fffec6a5b2" -> "0x00124b00120144ae" [penwidth=2, weight=1, color="#009900", label="92 (routes: 0x198c)"]
+              "0x000b57fffec6a5b3" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{bulb_color|0x000b57fffec6a5b3 (0x9dcf)|Philips Hue Go (7146060PH)|unknown}"];
               "0x000b57fffec6a5b3" -> "0x00124b00120144ae" [penwidth=0.5, weight=0, color="#994444", label="120"]
               "0x000b57fffec6a5b3" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="110"]
-              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (6538)|Xiaomi Aqara double key wireless wall switch (WXKG02LM)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45521" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{button_double_key|0x0017880104e45521 (0x198a)|Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)|9 seconds ago}"];
               "0x0017880104e45521" -> "0x0017880104e45559" [penwidth=1, weight=0, color="#994444", label="130"]
-              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (6536)failed: lqi,routingTable|Boef notSupportedModelID|1970-01-01T01:00:01+01:00}"];
-              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (6540)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|1970-01-01T01:00:01+01:00}"];
+              "0x0017880104e45525" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{0x0017880104e45525|0x0017880104e45525 (0x1988)failed: lqi,routingTable|Boef notSupportedModelID|9 seconds ago}"];
+              "0x0017880104e45559" [style="rounded, filled", fillcolor="#4ea3e0", fontcolor="#ffffff", label="{cc2530_router|0x0017880104e45559 (0x198c)|Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)|9 seconds ago}"];
               "0x0017880104e45559" -> "0x000b57fffec6a5b2" [penwidth=0.5, weight=0, color="#994444", label="100"]
+              "0x0017880104e45511" [style="rounded, dashed, filled", fillcolor="#fff8ce", fontcolor="#000000", label="{0x0017880104e45511|0x0017880104e45511 (0x045a)|external external (external_converter_device)|9 seconds ago}"];
+              "0x0017880104e45511" -> "0x00124b00120144ae" [penwidth=1, weight=0, color="#994444", label="92"]
             }`;
 
         const expectedLines = expected.split('\n');
@@ -165,7 +163,133 @@ describe('Networkmap', () => {
         for (let i = 0; i < expectedLines.length; i++) {
             expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
         }
-        Date.prototype.getTimezoneOffset = getTimezoneOffset;
-        Date.prototype.getHours = getHours;
+    });
+
+    it('Output plantuml networkmap legacy api', async () => {
+        mock();
+        const device = zigbeeHerdsman.devices.bulb_color;
+        device.lastSeen = null;
+        const endpoint = device.getEndpoint(1);
+        const data = {modelID: 'test'}
+        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        MQTT.events.message('zigbee2mqtt/bridge/networkmap/routes', 'plantuml');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/networkmap/plantuml');
+
+        const expected = `' paste into: https://www.planttext.com/
+
+        @startuml
+        card 0x0017880104e45511 [
+        0x0017880104e45511
+        ---
+        0x0017880104e45511 (0x045a)
+        ---
+        external external (external_converter_device)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x0017880104e45525 [
+        0x0017880104e45525
+        ---
+        0x0017880104e45525 (0x1988) failed: lqi,routingTable
+        ---
+        Boef notSupportedModelID
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b2 [
+        bulb
+        ---
+        0x000b57fffec6a5b2 (0x9db1)
+        ---
+        IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (LED1545G12)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x000b57fffec6a5b3 [
+        bulb_color
+        ---
+        0x000b57fffec6a5b3 (0x9dcf)
+        ---
+        Philips Hue Go (7146060PH)
+        ---
+        unknown
+        ]
+
+        card 0x0017880104e45521 [
+        button_double_key
+        ---
+        0x0017880104e45521 (0x198a)
+        ---
+        Xiaomi Aqara double key wireless wall switch (2016 model) (WXKG02LM_rev1)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x0017880104e45559 [
+        cc2530_router
+        ---
+        0x0017880104e45559 (0x198c)
+        ---
+        Custom devices (DiY) [CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/) (CC2530.ROUTER)
+        ---
+        9 seconds ago
+        ]
+
+        card 0x00124b00120144ae [
+        Coordinator
+        ---
+        0x00124b00120144ae (0x0000)
+        ---
+        0 seconds ago
+        ]
+
+        0x000b57fffec6a5b3 --> 0x00124b00120144ae: 120
+        0x000b57fffec6a5b2 --> 0x00124b00120144ae: 92
+        0x0017880104e45511 --> 0x00124b00120144ae: 92
+        0x000b57fffec6a5b3 --> 0x000b57fffec6a5b2: 110
+        0x0017880104e45559 --> 0x000b57fffec6a5b2: 100
+        0x0017880104e45521 --> 0x0017880104e45559: 130
+
+        @enduml`;
+
+        const expectedLines = expected.split('\n');
+        const actualLines = call[1].split('\n');
+
+        for (let i = 0; i < expectedLines.length; i++) {
+            expect(actualLines[i].trim()).toStrictEqual(expectedLines[i].trim());
+        }
+    });
+
+    it('Should output raw networkmap', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', stringify({type: 'raw', routes: true}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        let call = MQTT.publish.mock.calls[0];
+        expect(call[0]).toStrictEqual('zigbee2mqtt/bridge/response/networkmap');
+
+        const expected = {"data":{"routes":true,"type":"raw","value":{"links":[{"depth":1,"linkquality":120,"lqi":120,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[{"destinationAddress":6540,"nextHop":40369,"status":"ACTIVE"}],"source":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"sourceIeeeAddr":"0x000b57fffec6a5b2","sourceNwkAddr":40369,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":1,"linkquality":92,"lqi":92,"relationship":2,"routes":[],"source":{"ieeeAddr":"0x0017880104e45511","networkAddress":1114},"sourceIeeeAddr":"0x0017880104e45511","sourceNwkAddr":1114,"target":{"ieeeAddr":"0x00124b00120144ae","networkAddress":0},"targetIeeeAddr":"0x00124b00120144ae"},{"depth":2,"linkquality":110,"lqi":110,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x000b57fffec6a5b3","networkAddress":40399},"sourceIeeeAddr":"0x000b57fffec6a5b3","sourceNwkAddr":40399,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":100,"lqi":100,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"sourceIeeeAddr":"0x0017880104e45559","sourceNwkAddr":6540,"target":{"ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369},"targetIeeeAddr":"0x000b57fffec6a5b2"},{"depth":2,"linkquality":130,"lqi":130,"relationship":1,"routes":[],"source":{"ieeeAddr":"0x0017880104e45521","networkAddress":6538},"sourceIeeeAddr":"0x0017880104e45521","sourceNwkAddr":6538,"target":{"ieeeAddr":"0x0017880104e45559","networkAddress":6540},"targetIeeeAddr":"0x0017880104e45559"}],"nodes":[{"definition":null,"failed":[],"friendlyName":"Coordinator","ieeeAddr":"0x00124b00120144ae","lastSeen":1000,"modelID":null,"networkAddress":0,"type":"Coordinator"},{"definition":{"description":"TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white","model":"LED1545G12","supports":"light (state, brightness, color_temp, color_temp_startup), effect, power_on_behavior, linkquality","vendor":"IKEA"},"failed":[],"friendlyName":"bulb","ieeeAddr":"0x000b57fffec6a5b2","lastSeen":1000,"modelID":"TRADFRI bulb E27 WS opal 980lm","networkAddress":40369,"type":"Router"},{"definition":{"description":"Hue Go","model":"7146060PH","supports":"light (state, brightness, color_temp, color_temp_startup, color_xy, color_hs), effect, linkquality","vendor":"Philips"},"failed":[],"friendlyName":"bulb_color","ieeeAddr":"0x000b57fffec6a5b3","lastSeen":1000,"modelID":"LLC020","networkAddress":40399,"type":"Router"},{"definition":{"description":"Aqara double key wireless wall switch (2016 model)","model":"WXKG02LM_rev1","supports":"battery, action, voltage, linkquality","vendor":"Xiaomi"},"friendlyName":"button_double_key","ieeeAddr":"0x0017880104e45521","lastSeen":1000,"modelID":"lumi.sensor_86sw2.es1","networkAddress":6538,"type":"EndDevice"},{"definition":null,"failed":["lqi","routingTable"],"friendlyName":"0x0017880104e45525","ieeeAddr":"0x0017880104e45525","lastSeen":1000,"manufacturerName":"Boef","modelID":"notSupportedModelID","networkAddress":6536,"type":"Router"},{"definition":{"description":"[CC2530 router](http://ptvo.info/cc2530-based-zigbee-coordinator-and-router-112/)","model":"CC2530.ROUTER","supports":"led, linkquality","vendor":"Custom devices (DiY)"},"failed":[],"friendlyName":"cc2530_router","ieeeAddr":"0x0017880104e45559","lastSeen":1000,"modelID":"lumi.router","networkAddress":6540,"type":"Router"},{"definition":{"description":"external","model":"external_converter_device","supports":"linkquality","vendor":"external"},"friendlyName":"0x0017880104e45511","ieeeAddr":"0x0017880104e45511","lastSeen":1000,"modelID":"external_converter_device","networkAddress":1114,"type":"EndDevice"}]}},"status":"ok"};
+        const actual = JSON.parse(call[1]);
+        expect(actual).toStrictEqual(expected);
+    });
+
+    it('Should throw error when rquesting invalid type', async () => {
+        mock();
+        MQTT.publish.mockClear();
+        MQTT.events.message('zigbee2mqtt/bridge/request/networkmap', 'not_existing');
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/networkmap',
+            stringify({"data":{},"status":"error","error":"Type 'not_existing' not supported, allowed are: raw,graphviz,plantuml"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
     });
 });

test/onEvent.test.js

@@ -6,7 +6,7 @@ zigbeeHerdsman.returnDevices.push('0x0017880104e45560');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
@@ -22,14 +22,24 @@ describe('On event', () => {
     const device = zigbeeHerdsman.devices.LIVOLO;
 
     beforeEach(async () => {
+        jest.useFakeTimers();
         data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+        settings.reRead();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+        await flushPromises();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
         zigbeeHerdsmanConverters.onEvent.mockClear();
-        await flushPromises();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should call with start event', async () => {

test/otaUpdate.test.js

@@ -4,33 +4,325 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 describe('OTA update', () => {
     let controller;
 
-    mockClear = (mapped) => {
+    const mockClear = (mapped) => {
         mapped.ota.updateToLatest = jest.fn();
         mapped.ota.isUpdateAvailable = jest.fn();
     }
 
-    beforeEach(async () => {
+    beforeAll(async () => {
         data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+        settings.reRead();
+        data.writeDefaultConfiguration();
+        settings.set(['advanced', 'ikea_ota_use_test_url'], true);
+        settings.reRead();
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
+    });
+
+    afterEach(async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], false);
+        jest.runOnlyPendingTimers();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    });
+
+    beforeEach(async () => {
+        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
+        extension.lastChecked = {};
+        extension.inProgress = new Set();
+        controller.state.state = {};
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/check');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/ota_update/update');
+    it('Should OTA update a device', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        let count = 0;
+        endpoint.read.mockImplementation(() => {
+            count++;
+            return {swBuildId: count, dateCode: '2019010' + count}
+        });
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        logger.info.mockClear();
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            onUpdate(0, null);
+            onUpdate(10, 3600.2123);
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', 'bulb');
+        await flushPromises();
+        expect(logger.info).toHaveBeenCalledWith(`Updating 'bulb' to latest firmware`);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
+        expect(device.save).toHaveBeenCalledTimes(1);
+        expect(device.dateCode).toBe('20190102');
+        expect(device.softwareBuildID).toBe(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"updating","progress":0}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"updating","progress":10,"remaining":3600}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id": "bulb","from":{"software_build_id":1,"date_code":"20190101"},"to":{"software_build_id":2,"date_code":"20190102"}},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/devices',
+            expect.any(String),
+          { retain: true, qos: 0 },
+          expect.any(Function)
+        );
     });
 
-    it('Should OTA update a device', async () => {
+    it('Should handle when OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        device.save.mockClear();
+        mapped.ota.updateToLatest.mockImplementationOnce((a, b, onUpdate) => {
+            throw new Error('Update failed');
+        });
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', stringify({id: "bulb"}));
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":true,"update":{"state":"available"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id": "bulb"},"status":"error","error":"Update of 'bulb' failed (Update failed)"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should be able to check if OTA update is available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb","updateAvailable":false},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+
+        MQTT.publish.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(2);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb","updateAvailable":true},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should handle if OTA update check fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {throw new Error('RF singals disturbed because of dogs barking')});
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should fail when device does not exist', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "not_existing_deviceooo");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "not_existing_deviceooo"},"status":"error","error": `Device 'not_existing_deviceooo' does not exist`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "dimmer_wall_switch");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "dimmer_wall_switch"},"status":"error","error": `Device 'dimmer_wall_switch' does not support OTA updates`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should refuse to check/update when already in progress', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+
+        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
+            return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
+        });
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
+        await flushPromises();
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/check', "bulb");
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        jest.runOnlyPendingTimers();
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/check',
+            stringify({"data":{"id": "bulb"},"status":"error","error": `Update or check for update already in progress for 'bulb'`}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Shouldnt crash when read modelID before/after OTA update fails', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint = device.endpoints[0];
+        endpoint.read.mockImplementation(() => {throw new Error('Failed!')});
+
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/ota_update/update', "bulb");
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/ota_update/update',
+            stringify({"data":{"id":"bulb","from":null,"to":null},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should check for update when device requests it', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+
+        // Should not request again when device asks again after a short time
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+
+        logger.info.mockClear();
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":true,"update":{"state":"available"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should check for update when device requests it and it is not available', async () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        device.endpoints[0].commandResponse.mockClear();
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bulb',
+            stringify({"update_available":false,"update":{"state":"idle"}}),
+            {retain: true, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should not check for update when device requests it and disable_automatic_update_check is set to true', async () => {
+        settings.set(['ota', 'disable_automatic_update_check'], true);
+        const device = zigbeeHerdsman.devices.bulb;
+        const data = {imageType: 12382};
+        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
+        mockClear(mapped);
+        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.info.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
+        const device = zigbeeHerdsman.devices.HGZB04D;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
+    });
+
+    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
+        const device = zigbeeHerdsman.devices.SV01;
+        const data = {imageType: 12382};
+        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
+        logger.error.mockClear();
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
+        expect(logger.error).toHaveBeenCalledTimes(0);
+    });
+
+    it('Legacy api: Should OTA update a device', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         let count = 0;
@@ -55,15 +347,15 @@ describe('OTA update', () => {
         expect(mapped.ota.updateToLatest).toHaveBeenCalledTimes(1);
         expect(mapped.ota.updateToLatest).toHaveBeenCalledWith(device, logger, expect.any(Function));
         expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 0.00%`);
-        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, +- 60 minutes remaining`);
-        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"softwareBuildID":1,"dateCode":"20190101"}' to '{"softwareBuildID":2,"dateCode":"20190102"}'`);
+        expect(logger.info).toHaveBeenCalledWith(`Update of 'bulb' at 10.00%, ≈ 60 minutes remaining`);
+        expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb', from '{"dateCode":"20190101","softwareBuildID":1}' to '{"dateCode":"20190102","softwareBuildID":2}'`);
         expect(logger.error).toHaveBeenCalledTimes(0);
         expect(device.save).toHaveBeenCalledTimes(1);
         expect(device.dateCode).toBe('20190102');
         expect(device.softwareBuildID).toBe(2);
     });
 
-    it('Should handle when OTA update fails', async () => {
+    it('Legacy api: Should handle when OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         endpoint.read.mockImplementation(() => {return {swBuildId: 1, dateCode: '2019010'}});
@@ -82,7 +374,7 @@ describe('OTA update', () => {
         expect(logger.error).toHaveBeenCalledWith(`Update of 'bulb' failed (Update failed)`);
     });
 
-    it('Should be able to check if OTA update is available', async () => {
+    it('Legacy api: Should be able to check if OTA update is available', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -104,7 +396,7 @@ describe('OTA update', () => {
         expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
     });
 
-    it('Should handle if OTA update check fails', async () => {
+    it('Legacy api: Should handle if OTA update check fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const mapped = zigbeeHerdsmanConverters.findByDevice(device)
         mockClear(mapped);
@@ -118,33 +410,13 @@ describe('OTA update', () => {
         expect(logger.error).toHaveBeenCalledWith(`Failed to check if update available for 'bulb' (RF singals disturbed because of dogs barking)`);
     });
 
-    it('Should not check for OTA when device does not support it', async () => {
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'ZNLDP12LM');
+    it('Legacy api: Should not check for OTA when device does not support it', async () => {
+        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'dimmer_wall_switch');
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledWith(`Device 'ZNLDP12LM' does not support OTA updates`);
+        expect(logger.error).toHaveBeenCalledWith(`Device 'dimmer_wall_switch' does not support OTA updates`);
     });
 
-    it('Should refuse to check/update when already in progress', async () => {
-        jest.useFakeTimers();
-        const device = zigbeeHerdsman.devices.bulb;
-        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
-        mockClear(mapped);
-
-        logger.info.mockClear();
-        mapped.ota.isUpdateAvailable.mockImplementationOnce(() => {
-            return new Promise((resolve, reject) => {setTimeout(() => resolve(), 99999)})
-        });
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
-        await flushPromises();
-        MQTT.events.message('zigbee2mqtt/bridge/ota_update/check', 'bulb');
-        await flushPromises();
-        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith(`Update or check already in progress for 'bulb', skipping...`);
-        jest.runAllTimers();
-        await flushPromises();
-    });
-
-    it('Shouldnt crash when read modelID after OTA update fails', async () => {
+    it('Legacy api: Shouldnt crash when read modelID after OTA update fails', async () => {
         const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.endpoints[0];
         let count = 0;
@@ -161,55 +433,4 @@ describe('OTA update', () => {
         await flushPromises();
         expect(logger.info).toHaveBeenCalledWith(`Finished update of 'bulb'`);
     });
-
-    it('Should check for update when device requests it', async () => {
-        const device = zigbeeHerdsman.devices.bulb;
-        const data = {imageType: 12382};
-        const mapped = zigbeeHerdsmanConverters.findByDevice(device)
-        mockClear(mapped);
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(true);
-        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
-        logger.info.mockClear();
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledWith(device, logger, {"imageType": 12382});
-        expect(logger.info).toHaveBeenCalledWith(`Update available for 'bulb'`);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 0x95});
-
-        // Should not request again when device asks again after a short time
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(mapped.ota.isUpdateAvailable).toHaveBeenCalledTimes(1);
-
-        const extension = controller.extensions.find((e) => e.constructor.name === 'OTAUpdate');
-        extension.lastChecked = {};
-        logger.info.mockClear();
-        mapped.ota.isUpdateAvailable.mockReturnValueOnce(false);
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(logger.info).not.toHaveBeenCalledWith(`Update available for 'bulb'`)
-    });
-
-    it('Should respond with NO_IMAGE_AVAILABLE when not supporting OTA', async () => {
-        const device = zigbeeHerdsman.devices.QBKG04LM;
-        const data = {imageType: 12382};
-        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(1);
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledWith("genOta", "queryNextImageResponse", {"status": 152});
-    });
-
-    it('Shouldnt respond with NO_IMAGE_AVAILABLE when not supporting OTA and device has no OTA endpoint', async () => {
-        const device = zigbeeHerdsman.devices.QBKG03LM;
-        const data = {imageType: 12382};
-        const payload = {data, cluster: 'genOta', device, endpoint: device.getEndpoint(1), type: 'commandQueryNextImageRequest', linkquality: 10};
-        logger.error.mockClear();
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(device.endpoints[0].commandResponse).toHaveBeenCalledTimes(0);
-        expect(logger.error).toHaveBeenCalledTimes(0);
-    });
 });

test/receive.test.js

@@ -1,59 +1,67 @@
 const data = require('./stub/data');
 const logger = require('./stub/logger');
+const stringify = require('json-stable-stringify-without-jsonify');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 
 const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Receive', () => {
     let controller;
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
+    });
+
+    beforeEach(async () => {
+        controller.state.state = {};
+        data.writeDefaultConfiguration();
+        settings.reRead();
         mocksClear.forEach((m) => m.mockClear());
+        delete zigbeeHerdsman.devices.WXKG11LM.linkquality;
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
     });
 
     it('Should handle a zigbee message', async () => {
         const device = zigbeeHerdsman.devices.WXKG11LM;
+        device.linkquality = 10;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'single', linkquality: 10});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single', click: 'single', linkquality: 10}), {retain: false, qos: 0}, expect.any(Function));
     });
 
     it('Should handle a zigbee message which uses ep (left)', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'left', action: 'single_left'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should handle a zigbee message which uses ep (right)', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {onOff: 1}
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(2), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({click: 'right', action: 'single_right'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -63,14 +71,42 @@ describe('Receive', () => {
         const payload = {data, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.85});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
+    it('Should allow to invert cover', async () => {
+        const device = zigbeeHerdsman.devices.J1;
+
+        // Non-inverted (open = 100, close = 0)
+        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 10, tilt: 20}), {retain: false, qos: 0}, expect.any(Function));
+
+        // Inverted
+        MQTT.publish.mockClear();
+        settings.set(['devices', device.ieeeAddr, 'invert_cover'], true);
+        await zigbeeHerdsman.events.message({data: {currentPositionLiftPercentage: 90, currentPositionTiltPercentage: 80}, cluster: 'closuresWindowCovering', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/J1_cover', stringify({position: 90, tilt: 80}), {retain: false, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to disable the legacy integration', async () => {
+        const device = zigbeeHerdsman.devices.WXKG11LM;
+        settings.set(['devices', device.ieeeAddr, 'legacy'], false);
+        const data = {onOff: 1}
+        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/button', stringify({action: 'single'}), {retain: false, qos: 0}, expect.any(Function));
+    });
+
     it('Should debounce messages', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         const data1 = {measuredValue: 8}
@@ -85,16 +121,15 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
     it('Should debounce and retain messages when set via device_options', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['device_options', 'debounce'], 0.1);
         settings.set(['device_options', 'retain'], true);
@@ -111,16 +146,15 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, humidity: 0.01, pressure: 2});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": true});
     });
 
     it('Should debounce messages only with the same payload values for provided debounce_ignore keys', async () => {
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.WSDCGQ11LM;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         settings.set(['devices', device.ieeeAddr, 'debounce_ignore'], ['temperature']);
@@ -135,25 +169,24 @@ describe('Receive', () => {
         await flushPromises();
         jest.advanceTimersByTime(50);
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2, linkquality: 13});
-        jest.runAllTimers();
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: 0.08, pressure: 2});
+        jest.runOnlyPendingTimers();
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03, linkquality: 13});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({temperature: 0.07, pressure: 2, humidity: 0.03});
     });
 
     it('Shouldnt republish old state', async () => {
         // https://github.com/Koenkk/zigbee2mqtt/issues/3572
-        jest.useFakeTimers();
         const device = zigbeeHerdsman.devices.bulb;
         settings.set(['devices', device.ieeeAddr, 'debounce'], 0.1);
         await zigbeeHerdsman.events.message({data: {onOff: 0}, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10});
-        await MQTT.events.message('zigbee2mqtt/bulb/set', JSON.stringify({state: 'ON'}));
+        await MQTT.events.message('zigbee2mqtt/bulb/set', stringify({state: 'ON'}));
         await flushPromises();
-        jest.runAllTimers();
+        jest.runOnlyPendingTimers();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({state: 'ON'});
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toStrictEqual({state: 'ON'});
     });
 
     it('Should handle a zigbee message with 1 precision', async () => {
@@ -165,7 +198,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -178,7 +211,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -191,7 +224,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -0.8});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -205,7 +238,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/weather_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({temperature: -1});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 1, "retain": false});
     });
 
@@ -218,61 +251,61 @@ describe('Receive', () => {
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":915.5});
 
         MQTT.publish.mockClear();
         payload = {data: {"16":9354,"20":-1,"measuredValue":915}, cluster: 'msPressureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4,"linkquality":10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({"battery":91,"voltage":2985,"temperature":23.5,"humidity":46.08,"pressure":935.4});
     });
 
     it('Should handle a zigbee message with voltage 3010', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 3010}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 100, voltage: 3010});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should handle a zigbee message with voltage 2850', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2850}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 35, voltage: 2850});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should handle a zigbee message with voltage 2650', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2650}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 14, voltage: 2650});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
     it('Should handle a zigbee message with voltage 2000', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         const data = {'65281': {'1': 2000}}
         const payload = {data, cluster: 'genBasic', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({battery: 0, voltage: 2000});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -284,7 +317,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/occupancy_sensor');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'battery': 100, 'illuminance': 381, "illuminance_lux": 381, 'voltage': 3045, 'temperature': 19});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -297,8 +330,21 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
+    it('Should publish last_seen for unhandled messages', async () => {
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        const data = {onOff: 1};
+        const payload = {data, cluster: 'genRssiLocation', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/button_double_key');
+        expect(typeof JSON.parse(MQTT.publish.mock.calls[0][1]).last_seen).toBe('number')
+        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+    });
+
     it('Should publish last_seen epoch', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'epoch');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -311,7 +357,7 @@ describe('Receive', () => {
     });
 
     it('Should publish last_seen ISO_8601', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'ISO_8601');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -324,7 +370,7 @@ describe('Receive', () => {
     });
 
     it('Should publish last_seen ISO_8601_local', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
+        const device = zigbeeHerdsman.devices.WXKG02LM_rev1;
         settings.set(['advanced', 'last_seen'], 'ISO_8601_local');
         const data = {onOff: 1};
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
@@ -336,13 +382,18 @@ describe('Receive', () => {
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
-    it('Should not handle messages forwarded Xiaomi messages', async () => {
+    it('Should handle forwarded Xiaomi messages', async () => {
         const device = zigbeeHerdsman.devices.ZNCZ02LM;
-        const data = {onOff: 1};
-        const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 599};
+        const payload = {data: {measuredValue: -85}, cluster: 'msTemperatureMeasurement', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 6539};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/weather_sensor',
+            stringify({temperature: -0.85}),
+            {"qos": 1, "retain": false},
+            expect.any(Function),
+        )
     });
 
     it('Should handle messages from Xiaomi router devices', async () => {
@@ -351,10 +402,20 @@ describe('Receive', () => {
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 20};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/power_plug');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'state': 'ON', linkquality: 20});
-        expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
+        expect(MQTT.publish).toHaveBeenCalledTimes(2);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/power_plug',
+            stringify({state: 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/switch_group',
+            stringify({'state': 'ON'}),
+            { retain: false, qos: 0 },
+            expect.any(Function)
+        );
+
     });
 
     it('Should not handle messages from coordinator', async () => {
@@ -366,23 +427,27 @@ describe('Receive', () => {
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from unsupported devices', async () => {
+    it('Should not handle messages from unsupported devices and link to docs', async () => {
         const device = zigbeeHerdsman.devices.unsupported;
         const data = {onOff: 1};
+        logger.warn.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
+        expect(logger.warn).toHaveBeenCalledWith(`Received message from unsupported device with Zigbee model 'notSupportedModelID' and manufacturer name 'notSupportedMfg'`);
+        expect(logger.warn).toHaveBeenCalledWith(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
 
-    it('Should not handle messages from still interviewing devices with unknown modelID', async () => {
+    it('Should not handle messages from still interviewing devices with unknown definition', async () => {
         const device = zigbeeHerdsman.devices.interviewing;
         const data = {onOff: 1};
+        logger.debug.mockClear();
         const payload = {data, cluster: 'genOnOff', device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10};
         await zigbeeHerdsman.events.message(payload);
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, modelID is undefined and still interviewing`);
+        expect(logger.debug).toHaveBeenCalledWith(`Skipping message, definition is undefined and still interviewing`);
     });
 
     it('Should handle a command', async () => {
@@ -393,7 +458,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
     });
 
@@ -411,10 +476,10 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(MQTT.publish.mock.calls[0][2]).toStrictEqual({"qos": 0, "retain": false});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/ikea_onoff');
-        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop'});
+        expect(JSON.parse(MQTT.publish.mock.calls[1][1])).toMatchObject({'click': 'brightness_stop', action: 'brightness_stop'});
         expect(JSON.parse(MQTT.publish.mock.calls[1][1]).elapsed).toBe(50);
         expect(MQTT.publish.mock.calls[1][2]).toStrictEqual({"qos": 0, "retain": false});
         Date.now = oldNow;
@@ -440,7 +505,7 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_NEW');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 0.66, power: 49.6});
 
         MQTT.publish.mockClear();
         const SP600_OLD = zigbeeHerdsman.devices.SP600_OLD;
@@ -448,6 +513,6 @@ describe('Receive', () => {
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(1);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/SP600_OLD');
-        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496, linkquality: 10});
+        expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({energy: 6.648, power: 496});
     });
 });

test/settings.test.js

@@ -1,37 +1,53 @@
 require('./stub/logger');
 require('./stub/data');
 const data = require('../lib/util/data.js');
-const utils = require('../lib/util/utils.js');
-const settings = require('../lib/util/settings.js');
+const utils = require('../lib/util/utils');
+const settings = require('../lib/util/settings.ts');
 const fs = require('fs');
 const configurationFile = data.joinPath('configuration.yaml');
 const devicesFile = data.joinPath('devices.yaml');
+const devicesFile2 = data.joinPath('devices2.yaml');
 const groupsFile = data.joinPath('groups.yaml');
 const secretFile = data.joinPath('secret.yaml');
 const yaml = require('js-yaml');
+const objectAssignDeep = require(`object-assign-deep`);
+
+const minimalConfig = {
+    permit_join: true,
+    homeassistant: true,
+    mqtt: {base_topic: 'zigbee2mqtt', server: 'localhost'},
+};
 
 describe('Settings', () => {
     const write = (file, json, reread=true) => {
-        fs.writeFileSync(file, yaml.safeDump(json))
+        fs.writeFileSync(file, yaml.dump(json))
         if (reread) {
-            settings._reRead();
+            settings.reRead();
         }
     };
-    const read = (file) => yaml.safeLoad(fs.readFileSync(file, 'utf8'));
+    const read = (file) => yaml.load(fs.readFileSync(file, 'utf8'));
     const remove = (file) => {
         if (fs.existsSync(file)) fs.unlinkSync(file);
     }
+    const clearEnvironmentVariables = () => {
+        Object.keys(process.env).forEach((key) => {
+            if(key.indexOf('ZIGBEE2MQTT_CONFIG_') >= 0) {
+                delete process.env[key];
+            }
+        });
+    }
 
     beforeEach(() => {
         remove(configurationFile);
         remove(devicesFile);
         remove(groupsFile);
+        clearEnvironmentVariables();
     });
 
     it('Should return default settings', () => {
         write(configurationFile, {});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expect(s).toStrictEqual(expected);
@@ -40,13 +56,36 @@ describe('Settings', () => {
     it('Should return settings', () => {
         write(configurationFile, {permit_join: true});
         const s = settings.get();
-        const expected = settings._getDefaults();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
         expected.devices = {};
         expected.groups = {};
         expected.permit_join = true;
         expect(s).toStrictEqual(expected);
     });
 
+    it('Should apply environment variables', () => {
+        process.env['ZIGBEE2MQTT_CONFIG_SERIAL_DISABLE_LED'] = 'true';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_SOFT_RESET_TIMEOUT'] = 1;
+        process.env['ZIGBEE2MQTT_CONFIG_EXPERIMENTAL_OUTPUT'] = 'csvtest';
+        process.env['ZIGBEE2MQTT_CONFIG_ADVANCED_AVAILABILITY_BLOCKLIST'] = '["0x43597f0dac781b1e", "x223b0aef2ae8d1b0"]';
+        process.env['ZIGBEE2MQTT_CONFIG_MAP_OPTIONS_GRAPHVIZ_COLORS_FILL'] = '{"enddevice": "#ff0000", "coordinator": "#00ff00", "router": "#0000ff"}';
+        process.env['ZIGBEE2MQTT_CONFIG_MQTT_BASE_TOPIC'] = 'testtopic';
+
+        write(configurationFile, {});
+        const s = settings.get();
+        const expected = objectAssignDeep.noMutate({}, settings.testing.defaults);
+        expected.devices = {};
+        expected.groups = {};
+        expected.serial.disable_led = true;
+        expected.advanced.soft_reset_timeout = 1;
+        expected.experimental.output = 'csvtest';
+        expected.advanced.availability_blocklist = ['0x43597f0dac781b1e', 'x223b0aef2ae8d1b0'];
+        expected.map_options.graphviz.colors.fill = {enddevice: '#ff0000', coordinator: '#00ff00', router: '#0000ff'};
+        expected.mqtt.base_topic = 'testtopic';
+
+        expect(s).toStrictEqual(expected);
+    });
+
     it('Should add devices', () => {
         write(configurationFile, {});
         settings.addDevice('0x12345678');
@@ -115,6 +154,7 @@ describe('Settings', () => {
 
         const expected = {
             include_device_information: false,
+            force_disable_retain: false,
             password: "mysecretpassword",
             server: "my.mqtt.server",
             user: "mysecretusername",
@@ -123,7 +163,7 @@ describe('Settings', () => {
         expect(settings.get().mqtt).toStrictEqual(expected);
         expect(settings.get().advanced.network_key).toStrictEqual([1,2,3]);
 
-        settings._write();
+        settings.testing.write();
         expect(read(configurationFile)).toStrictEqual(contentConfiguration);
         expect(read(secretFile)).toStrictEqual(contentSecret);
 
@@ -158,6 +198,32 @@ describe('Settings', () => {
         expect(device).toStrictEqual(expected);
     });
 
+    it('Should read devices form 2 separate files', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+        expect(settings.getDevice('0x12345678').friendly_name).toStrictEqual('0x12345678');
+        expect(settings.getDevice('0x87654321').friendly_name).toStrictEqual('0x87654321');
+    });
+
     it('Should add devices to a separate file', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -190,6 +256,47 @@ describe('Settings', () => {
         expect(read(devicesFile)).toStrictEqual(expected);
     });
 
+    it('Should add devices for first file when using 2 separates file', () => {
+        const contentConfiguration = {
+            devices: ['devices.yaml', 'devices2.yaml']
+        };
+
+        const contentDevices = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+        };
+
+        const contentDevices2 = {
+            '0x87654321': {
+                friendly_name: '0x87654321',
+                retain: false,
+            },
+        };
+
+        write(configurationFile, contentConfiguration);
+        write(devicesFile, contentDevices);
+        write(devicesFile2, contentDevices2);
+
+        settings.addDevice('0x1234');
+
+        expect(read(configurationFile)).toStrictEqual({devices: ['devices.yaml', 'devices2.yaml']});
+
+        const expected = {
+            '0x12345678': {
+                friendly_name: '0x12345678',
+                retain: false,
+            },
+            '0x1234': {
+                friendly_name: '0x1234',
+            },
+        };
+
+        expect(read(devicesFile)).toStrictEqual(expected);
+        expect(read(devicesFile2)).toStrictEqual(contentDevices2);
+    });
+
     it('Should add devices to a separate file if devices.yaml doesnt exist', () => {
         const contentConfiguration = {
             devices: 'devices.yaml',
@@ -245,7 +352,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -271,7 +377,6 @@ describe('Settings', () => {
             friendlyName: '123',
             friendly_name: '123',
             devices: [],
-            optimistic: true,
         };
 
         expect(group).toStrictEqual(expected);
@@ -313,7 +418,6 @@ describe('Settings', () => {
 
         const group = settings.getGroup('1');
         const expectedGroup = {
-            optimistic: true,
             ID: 1,
             friendlyName: '123',
             friendly_name: '123',
@@ -359,6 +463,14 @@ describe('Settings', () => {
         expect(settings.get().groups).toStrictEqual(expected);
     });
 
+    it('Should throw error when changing entity options of non-existing device', () => {
+        write(configurationFile, {});
+
+        expect(() => {
+            settings.changeEntityOptions('not_existing_123', {});
+        }).toThrow(new Error("Device or group 'not_existing_123' does not exist"));
+    });
+
     it('Should not add duplicate groups', () => {
         write(configurationFile, {});
 
@@ -381,7 +493,7 @@ describe('Settings', () => {
         settings.addGroup('test123', 123);
         expect(() => {
             settings.addGroup('test_id_123', 123);
-        }).toThrow(new Error("group id '123' is already in use"));
+        }).toThrow(new Error("Group ID '123' is already in use"));
         const expected = {
             '123': {
                 friendly_name: 'test123',
@@ -497,40 +609,75 @@ describe('Settings', () => {
 
     it('Should not allow any string values for network_key', () => {
         write(configurationFile, {
+            ...minimalConfig,
             advanced: {network_key: 'NOT_GENERATE'},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`);
+        const error = `advanced.network_key: should be array or 'GENERATE' (is 'NOT_GENERATE')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Should not allow any string values for pan_id', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            advanced: {pan_id: 'NOT_GENERATE'},
+        });
+
+        settings.reRead();
+
+        const error = `advanced.pan_id: should be number or 'GENERATE' (is 'NOT_GENERATE')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Should allow retention configuration with MQTT v5', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            mqtt: {base_topic: 'zigbee2mqtt', server: 'localhost', version: 5},
+            devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
+        });
+
+        settings.reRead();
+        expect(settings.validate()).toEqual([]);
     });
 
 
     it('Should not allow retention configuration without MQTT v5', () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'tain', retention: 900}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError('MQTT retention requires protocol version 5');
+        const error = 'MQTT retention requires protocol version 5';
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
-    it('Should not allow non-existing entities in availability blacklist', () => {
+    it('Should not allow non-existing entities in availability_blocklist', () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'tain'}},
-            advanced: {availability_blacklist: ['0x0017880104e45519', 'non_existing']},
+            advanced: {availability_blocklist: ['0x0017880104e45519', 'non_existing']},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Non-existing entity 'non_existing' specified in 'availability_blacklist'`);
+        const error = `Non-existing entity 'non_existing' specified in 'availability_blocklist'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Validate should if settings does not conform to scheme', () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            advanced: null,
+        });
+
+        settings.reRead();
+
+        const error = `advanced must be object`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should ban devices', () => {
@@ -541,80 +688,114 @@ describe('Settings', () => {
         expect(settings.get().ban).toStrictEqual(['0x123', '0x1234']);
     });
 
+    it('Should add devices to blocklist', () => {
+        write(configurationFile, {});
+        settings.blockDevice('0x123');
+        expect(settings.get().blocklist).toStrictEqual(['0x123']);
+        settings.blockDevice('0x1234');
+        expect(settings.get().blocklist).toStrictEqual(['0x123', '0x1234']);
+    });
+
     it('Should throw error when yaml file is invalid', () => {
         fs.writeFileSync(configurationFile, `
              good: 9
              \t wrong
         `)
 
-        let error;
-        try {
-            settings._reRead();
-        } catch (e) {
-            error = e;
-        }
-
-        expect(error.message).toContain(`Your YAML file '${configurationFile}' is invalid`);
+        settings.testing.clear();
+        const error = `Your YAML file: '${configurationFile}' is invalid (use https://jsonformatter.org/yaml-validator to find and fix the issue)`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Should throw error when yaml file does not exist', () => {
-        let error;
-        try {
-            settings._reRead();
-        } catch (e) {
-            error = e;
-        }
-
-        expect(error.message).toContain(`no such file or directory, open`);
+        settings.testing.clear();
+        const error = `ENOENT: no such file or directory, open '${configurationFile}'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname', retain: false}},
             groups: {'1': {friendly_name: 'myname', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Duplicate friendly_name 'myname' found`);
+        const error = `Duplicate friendly_name 'myname' found`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Configuration friendly name cannot be empty', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: '', retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name must be at least 1 char long`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Configuration friendly name cannot end with /', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/', retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name is not allowed to end or start with /`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
+    });
+
+    it('Configuration friendly name cannot contain null char', async () => {
+        write(configurationFile, {
+            ...minimalConfig,
+            devices: {'0x0017880104e45519': {friendly_name: 'blaa/blaa' + String.fromCharCode(0), retain: false}},
+        });
+
+        settings.reRead();
+
+        const error = `friendly_name is not allowed to contain null char`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name ends with /DIGIT', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname/123', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Friendly name cannot end with a "/DIGIT" ('myname/123')`);
+        const error = `Friendly name cannot end with a "/DIGIT" ('myname/123')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name contains a MQTT wildcard', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'myname#', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`);
+        const error = `MQTT wildcard (+ and #) not allowed in friendly_name ('myname#')`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when friendly_name is a postfix', async () => {
         write(configurationFile, {
+            ...minimalConfig,
             devices: {'0x0017880104e45519': {friendly_name: 'left', retain: false}},
         });
 
-        settings._reRead();
+        settings.reRead();
 
-        expect(() => {
-            settings.validate();
-        }).toThrowError(`Following friendly_name are not allowed: '${utils.getEndpointNames()}'`);
+        const error = `Following friendly_name are not allowed: '${utils.endpointNames}'`;
+        expect(settings.validate()).toEqual(expect.arrayContaining([error]));
     });
 
     it('Configuration shouldnt be valid when duplicate friendly_name are used', async () => {
@@ -625,7 +806,7 @@ describe('Settings', () => {
             },
         });
 
-        settings._reRead();
+        settings.reRead();
 
         expect(() => {
             settings.changeFriendlyName('myname1', 'myname');
@@ -640,7 +821,7 @@ describe('Settings', () => {
             },
         });
 
-        settings._reRead();
+        settings.reRead();
 
         expect(() => {
             settings.removeDevice('myname33');

test/stub/data.js

@@ -2,9 +2,11 @@ const tmp = require('tmp');
 const yaml = require('../../lib/util/yaml');
 const path = require('path');
 const fs = require('fs');
+const stringify = require('json-stable-stringify-without-jsonify');
 
 const mockDir = tmp.dirSync().name;
 const mockDirStorage = tmp.dirSync().name;
+const stateFile = path.join(mockDir, 'state.json');
 
 function writeDefaultConfiguration() {
     const config = {
@@ -108,7 +110,7 @@ function writeDefaultConfiguration() {
             },
             '0x0017880104e45551': {
                 retain: false,
-                friendly_name: 'smart_vent'
+                friendly_name: 'smart vent'
             },
             '0x0017880104e45552': {
                 retain: false,
@@ -153,6 +155,24 @@ function writeDefaultConfiguration() {
             },
             '0x0017880104e44559': {
                 friendly_name: '3157100_thermostat',
+            },
+            '0x0017880104a44559': {
+                friendly_name: 'J1_cover',
+            },
+            '0x0017882104a44559': {
+                friendly_name: 'TS0601_thermostat',
+            },
+            '0x0017882194e45543': {
+                friendly_name: 'QS-Zigbee-D02-TRIAC-2C-LN',
+            },
+            '0x0017880104e45724': {
+                friendly_name: 'GLEDOPTO_2ID',
+            },
+            '0x0017880104e45561': {
+                friendly_name: 'temperature_sensor',
+            },
+            '0x0017880104e45562': {
+                friendly_name: 'heating_actuator',
             }
         },
         groups: {
@@ -174,34 +194,63 @@ function writeDefaultConfiguration() {
                 retain: false,
                 devices: ['bulb_2']
             },
-        }
+            '12': {
+                friendly_name: 'thermostat_group',
+                retain: false,
+                devices: ['TS0601_thermostat'],
+            },
+            '14': {
+                friendly_name: 'switch_group',
+                retain: false,
+                devices: ['power_plug', 'bulb_2'],
+            },
+            '21': {
+                friendly_name: 'gledopto_group',
+                devices: ['GLEDOPTO_2ID/cct'],
+            },
+            '9': {
+                friendly_name: 'ha_discovery_group',
+                devices: ['bulb_color_2', 'bulb_2', 'wall_switch_double/right']
+            },
+        },
+        external_converters: [],
     };
 
     yaml.writeIfChanged(path.join(mockDir, 'configuration.yaml'), config);
 }
 
 function writeEmptyState() {
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify({}));
-}
-
-function writeDefaultState() {
-    const state = {
-        "0x000b57fffec6a5b2": {
-            "state": "ON",
-            "brightness": 50,
-            "color_temp": 370,
-            "linkquality": 99,
-        },
-        "0x0017880104e45517": {
-            "brightness": 255
-        },
-    }
-
-    fs.writeFileSync(path.join(mockDir, 'state.json'), JSON.stringify(state));
+    fs.writeFileSync(stateFile, stringify({}));
 }
 
 function removeState() {
-    fs.unlinkSync(path.join(mockDir, 'state.json'))
+    if (stateExists()) {
+        fs.unlinkSync(stateFile);
+    }
+}
+
+function stateExists() {
+    return fs.existsSync(stateFile);
+}
+
+const defaultState = {
+    "0x000b57fffec6a5b2": {
+        "state": "ON",
+        "brightness": 50,
+        "color_temp": 370,
+        "linkquality": 99,
+    },
+    "0x0017880104e45517": {
+        "brightness": 255
+    },
+}
+
+function getDefaultState() {
+    return defaultState;
+}
+
+function writeDefaultState() {
+    fs.writeFileSync(path.join(mockDir, 'state.json'), stringify(defaultState));
 }
 
 jest.mock('../../lib/util/data', () => ({
@@ -219,4 +268,6 @@ module.exports = {
     writeDefaultState,
     removeState,
     writeEmptyState,
+    stateExists,
+    getDefaultState,
 };

test/stub/logger.js

@@ -1,14 +1,28 @@
 let level = 'info';
 
+const transports = [];
+
+let transportsEnabled = false;
+const callTransports = (level, message) => {
+    if (transportsEnabled) {
+        for (const transport of transports) {
+            transport.log({level, message}, () => {});
+        }
+    }
+}
+
 const mock = {
-    info: jest.fn(),
-    warn: jest.fn(),
-    error: jest.fn(),
-    debug: jest.fn(),
-    trace: jest.fn(),
+    info: jest.fn().mockImplementation((msg) => callTransports('info', msg)),
+    warn: jest.fn().mockImplementation((msg) => callTransports('warn', msg)),
+    error: jest.fn().mockImplementation((msg) => callTransports('error', msg)),
+    debug: jest.fn().mockImplementation((msg) => callTransports('debug', msg)),
     cleanup: jest.fn(),
+    logOutput: jest.fn(),
+    add: (transport) => transports.push(transport),
+    addTransport: (transport) => transports.push(transport),
     setLevel: (newLevel) => {level = newLevel},
     getLevel: () => level,
+    setTransportsEnabled: (value) => {transportsEnabled = value},
 };
 
 jest.mock('../../lib/util/logger', () => (mock));