1.22.0

Co-authored-by: Koenkk <Koenkk@users.noreply.github.com>

.dockerignore

@@ -1,8 +1,9 @@
 .eslintignore
-.eslintrc.json
+.eslintrc.js
 .git
 .github
 .gitignore
+.idea
 .npmrc
 .travis*
 .travis.yml
@@ -12,4 +13,5 @@ images
 node_modules
 scripts
 test
-update.sh
+coverage
+update.sh

.eslintrc.js

@@ -0,0 +1,45 @@
+
+module.exports = {
+    'env': {
+        'jest/globals': true,
+        'es6': true,
+        'node': true,
+    },
+    'extends': ['eslint:recommended', 'google', 'plugin:jest/recommended', 'plugin:jest/style'],
+    'parserOptions': {
+        'ecmaVersion': 2018,
+        'sourceType': 'module',
+    },
+    'rules': {
+        'require-jsdoc': 'off',
+        'indent': ['error', 4],
+        'max-len': ['error', {'code': 120}],
+        'no-prototype-builtins': 'off',
+    },
+    'plugins': [
+        'jest',
+    ],
+    'overrides': [{
+        files: ['*.ts'],
+        parser: '@typescript-eslint/parser',
+        plugins: ['@typescript-eslint'],
+        extends: ['plugin:@typescript-eslint/recommended'],
+        parserOptions: {
+            project: './tsconfig.json',
+        },
+        rules: {
+            '@typescript-eslint/await-thenable': 'error',
+            '@typescript-eslint/ban-ts-comment': 'off',
+            '@typescript-eslint/explicit-function-return-type': 'error',
+            '@typescript-eslint/no-empty-function': 'off',
+            '@typescript-eslint/no-explicit-any': 'error',
+            '@typescript-eslint/no-unused-vars': 'error',
+            '@typescript-eslint/semi': ['error'],
+            'array-bracket-spacing': ['error', 'never'],
+            'indent': ['error', 4],
+            'max-len': ['error', {'code': 120}],
+            'no-return-await': 'error',
+            'object-curly-spacing': ['error', 'never'],
+        },
+    }],
+};

.eslintrc.json

@@ -1,21 +0,0 @@
-{
-    "env": {
-        "jest/globals": true,
-        "es6": true,
-        "node": true
-    },
-    "extends": ["eslint:recommended", "google", "plugin:jest/recommended", "plugin:jest/style"],
-    "parserOptions": {
-        "ecmaVersion": 2018,
-        "sourceType": "module"
-    },
-    "rules": {
-        "require-jsdoc": "off",
-        "indent": ["error", 4],
-        "max-len": ["error", { "code": 120 }],
-        "no-prototype-builtins": "off"
-    },
-    "plugins": [
-        "jest"
-    ]
-}

.github/FUNDING.yml

@@ -1,2 +1,4 @@
 github: [koenkk]
-custom: https://www.paypal.me/koenkk
+custom:
+  - https://www.paypal.me/koenkk
+  - https://www.buymeacoffee.com/koenkk

.github/ISSUE_TEMPLATE/config.yml

@@ -1,11 +1,17 @@
 blank_issues_enabled: false
 contact_links:
+  - name: Questions/discussion
+    url: https://github.com/Koenkk/zigbee2mqtt/discussions/new
+    about: Ask questions, discuss about devices or show things you made
+  - name: Frontend issues
+    url: https://github.com/nurikk/z2m-frontend/issues
+    about: Issues related to the frontend.
   - name: Home Assistant addon issues
-    url: https://github.com/danielwelch/hassio-zigbee2mqtt/issues
+    url: https://github.com/zigbee2mqtt/hassio-zigbee2mqtt/issues
     about: Issues related to the Home Assistant addon.
   - name: FAQ
     url: https://www.zigbee2mqtt.io/information/FAQ.html
     about: Frequently asked questions.
   - name: Support Chat
     url: https://discord.gg/NyseBeK
-    about: Chat for feedback, questions and troubleshooting.
+    about: Chat for feedback, questions and troubleshooting.

.github/ISSUE_TEMPLATE/new_device_support.md

@@ -8,7 +8,10 @@ assignees: ''
 ---
 
 <!--
-IMPORTANT: Before submitting a request, first follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+VERY IMPORTANT: Before submitting a request, first (try to) follow: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html
+
+Dumping just the info about this device without actually attemping adding support yourself will likely lead into this issue being ignored.
+Clearly indicate what you tried and where you got stuck.
 -->
 
 ## Information about the device + link

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
         registry-url: https://registry.npmjs.org/
     - name: Get NodeJS Version
       id: nodeversion
@@ -24,6 +24,8 @@ jobs:
     - name: Install dependencies
       if: steps.cache-node-modules.outputs.cache-hit != 'true'
       run: npm ci
+    - name: Build
+      run: npm run build
     - name: Test
       run: npm run test-with-coverage
     - name: Lint
@@ -31,46 +33,52 @@ jobs:
     - name: Docker login
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
       run: echo ${{ secrets.DOCKER_KEY }} | docker login -u koenkk --password-stdin
-    - name: Docker setup
+    - name: Docker login ghcr.io
       if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
-      run: |
-        sudo wget -O /usr/local/bin/buildx https://github.com/docker/buildx/releases/download/v0.3.1/buildx-v0.3.1.linux-amd64
-        sudo chmod a+x /usr/local/bin/buildx
-        docker run --rm --privileged hypriot/qemu-register:v2.7.0
-        buildx create --use
-        buildx ls
+      run: echo ${{ secrets.CR_PAT }} | docker login ghcr.io -u koenkk --password-stdin
+    - name: Docker setup - QEMU
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      uses: docker/setup-qemu-action@v1
+      with:
+        platforms: all
+    - name: Docker setup - Buildx
+      if: (github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) && github.event_name == 'push'
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+      with:
+        version: latest
     - name: Docker build dev
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
-          -t koenkk/zigbee2mqtt:latest-dev \
+          -t koenkk/zigbee2mqtt:latest-dev -t ghcr.io/koenkk/zigbee2mqtt:latest-dev \
           .
     - name: Docker build release
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
       run: |
         TAG="$(git describe --tags)"
-        buildx build \
+        docker buildx build \
           --build-arg COMMIT=$(git rev-parse --short HEAD) \
-          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7,linux/386 \
+          --platform linux/arm64/v8,linux/amd64,linux/arm/v6,linux/arm/v7 \
           -f docker/Dockerfile \
           --push \
-          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" \
+          -t koenkk/zigbee2mqtt:latest -t "koenkk/zigbee2mqtt:$TAG" -t ghcr.io/koenkk/zigbee2mqtt:latest -t "ghcr.io/koenkk/zigbee2mqtt:$TAG" \
           .
     - name: Publish to npm
       if: startsWith(github.ref, 'refs/tags/') && github.event_name == 'push'
       run: npm publish
       env:
         NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN }}
-    - name: Trigger danielwelch/hassio-zigbee2mqtt build
+    - name: Trigger zigbee2mqtt/hassio-zigbee2mqtt build
       if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
       run: |
         curl -s \
           --request POST \
           --header "Content-Type: application/json" \
-          --data '{"definition": {"id": 1}}' \
+          --data '{"definition": {"id": 9}}' \
           -u ${{ secrets.HASSIO_TRIGGER_SECRET }} \
-          "https://dev.azure.com/danielwelch2101/hassio-zigbee2mqtt/_apis/build/builds?api-version=5.1"
+          "https://dev.azure.com/zigbee2mqtt/Zigbee2mqtt%20Add-on/_apis/build/builds?api-version=5.1"

.github/workflows/stale.yml

@@ -16,3 +16,4 @@ jobs:
         days-before-stale: 30
         days-before-close: 7
         operations-per-run: 500
+        exempt-issue-labels: 'breaking change'

.github/workflows/update_deps.yml

@@ -14,11 +14,11 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
-    - run: npm install
-    - run: npm ls --depth 0 --prod --parseable | tail -n +2 | grep -v "zigbee" | xargs -I '{}' basename "{}" | xargs -I '{}' npm install '{}'@latest
-    - run: rm -rf node_modules npm-shrinkwrap.json && npm install && npm shrinkwrap --dev
-    - uses: peter-evans/create-pull-request@v2
+        node-version: 14
+    - run: rm -rf npm-shrinkwrap.json
+    - run: npx npm-check-updates -u
+    - run: npm install && npm shrinkwrap --dev
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update dependencies
         branch: "update_deps"

.github/workflows/update_frontend.yml

@@ -13,9 +13,9 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee2mqtt-frontend@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee2mqtt-frontend to ${{ github.event.client_payload.version }}
         branch: "update_frontend"

.github/workflows/update_zh.yml

@@ -13,9 +13,9 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee-herdsman@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman to ${{ github.event.client_payload.version }}
         branch: "update_zh"

.github/workflows/update_zhc.yml

@@ -13,9 +13,9 @@ jobs:
         ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
     - uses: actions/setup-node@v2-beta
       with:
-        node-version: 12
+        node-version: 14
     - run: npm install zigbee-herdsman-converters@${{ github.event.client_payload.version }} --save-exact
-    - uses: peter-evans/create-pull-request@v2
+    - uses: peter-evans/create-pull-request@v3
       with:
         commit-message: Update zigbee-herdsman-converters to ${{ github.event.client_payload.version }}
         branch: "update_zhc"

.gitignore

@@ -11,6 +11,9 @@ pids
 *.seed
 *.pid.lock
 
+# Compiled source
+dist/*
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
@@ -70,4 +73,10 @@ data/database.db.backup
 data/extension
 
 # MacOS indexing file
-.DS_Store
+.DS_Store
+
+# IDE specific folders
+.idea
+
+# Ignore config
+data/*.yaml

.npmignore

@@ -7,6 +7,7 @@ coverage
 .jenkins.yml
 .codeclimate.yml
 .github
+babel.config.js
 
 #linters
 .jscsrc
@@ -14,11 +15,13 @@ coverage
 .eslintrc*
 .dockerignore
 .eslintignore
+.gitignore
 
 #editor settings
 .vscode
 
 #src
+lib
 docs
 docker
 images

.vscode/settings.json

@@ -1,5 +0,0 @@
-{
-    "files.associations": {
-        "*.yaml": "home-assistant"
-    }
-}

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing to Zigbee2MQTT
+
+Everybody is invited and welcome to contribute to Zigbee2MQTT. Zigbee2MQTT is writen in JavaScript and is based upon [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) and [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters). Zigbee-herdsman-converters contains all device definition, zigbee-herdsman is responsible for handling all communication with the adapter.
+
+- Pull requests are always created against the [**dev**](https://github.com/Koenkk/zigbee2mqtt/tree/dev) branch.
+- Easiest way to start developing Zigbee2MQTT is by setting up a development environment (aka bare-metal installation). You can follow this [guide](https://www.zigbee2mqtt.io/getting_started/running_zigbee2mqtt.html#running-zigbee2mqtt) to do this.
+- You can run the tests locally by executing `npm test`. Zigbee2MQTT enforces 100% code coverage, in case you add new code check if you code is covered by running `npm run test-with-coverage`. The coverage report can be found under `coverage/lcov-report/index.html`. Linting is also enforced and can be run with `npm run eslint`.
+- When you want to add support for a new devices no changes to Zigbee2MQTT have to be made, only to zigbee-herdsman-converters. You can find a guide for it [here](https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html).

README.md

@@ -53,7 +53,7 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/7738048/40914297-49e6e560-6800-11e8-8904-36cce896e5a8.png">
 
 ### [Home Assistant](https://www.home-assistant.io/)
-- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/danielwelch/hassio-zigbee2mqtt) from [danielwelch](https://github.com/danielwelch)
+- [Home Assistant](https://www.home-assistant.io/hassio/): Using [the official addon](https://github.com/zigbee2mqtt/hassio-zigbee2mqtt)
 - Generic install or Hassbian: Using instructions [here](https://www.zigbee2mqtt.io/integration/home_assistant.html)
 
 <img align="left" height="100px" width="100px" src="https://user-images.githubusercontent.com/2734836/47615848-b8dd8700-dabd-11e8-9d77-175002dd8987.png">
@@ -67,7 +67,10 @@ Zigbee2MQTT integrates well with (almost) every home automation solution because
 ![Architecture](images/architecture.png)
 
 ### Internal Architecture
-Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/07-5123-06-zigbee-cluster-library-specification.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+Zigbee2MQTT is made up of three modules, each developed in its own Github project. Starting from the hardware (adapter) and moving up; [zigbee-herdsman](https://github.com/koenkk/zigbee-herdsman) connects to your Zigbee adapter an makes an API available to the higher levels of the stack. For e.g. Texas Instruments hardware, zigbee-herdsman uses the [TI zStack monitoring and test API](https://github.com/koenkk/zigbee-herdsman/raw/master/docs/Z-Stack%20Monitor%20and%20Test%20API.pdf) to communicate with the adapter. Zigbee-herdsman handles the core Zigbee communication. The module [zigbee-herdsman-converters](https://github.com/koenkk/zigbee-herdsman-converters) handles the mapping from individual device models to the Zigbee clusters they support. [Zigbee clusters](https://github.com/Koenkk/zigbee-herdsman/raw/master/docs/Zigbee%20Cluster%20Library%20Specification%20v7.pdf) are the layers of the Zigbee protocol on top of the base protocol that define things like how lights, sensors and switches talk to each other over the Zigbee network.  Finally, the Zigbee2MQTT module drives zigbee-herdsman and maps the zigbee messages to MQTT messages. Zigbee2MQTT also keeps track of the state of the system.  It uses a `database.db` file to store this state; a text file with a JSON database of connected devices and their capabilities.
+
+### Developing
+Zigbee2MQTT uses TypeScript (partially for now). Therefore after making changes to files in the `lib/` directory you need to recompile Zigbee2MQTT. This can be done by executing `npm run build`. For faster development instead of running `npm run build` you can run `npm run build-watch` in another terminal session, this will recompile as you change files.
 
 ## Supported devices
 See [Supported devices](https://www.zigbee2mqtt.io/information/supported_devices.html) to check whether your device is supported. There is quite an extensive list, including devices from vendors like Xiaomi, Ikea, Philips, OSRAM and more.

babel.config.js

@@ -0,0 +1,11 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+    assumptions: {setPublicClassFields: true},
+    plugins: [
+        ['@babel/plugin-proposal-decorators', {'legacy': true}],
+        ['@babel/plugin-proposal-class-properties'],
+    ],
+};

cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+const path = require('path');
+process.env['ZIGBEE2MQTT_DATA'] = process.env['ZIGBEE2MQTT_DATA'] || path.join(process.env['HOME'], '.z2m');
+require('./index');

docker/Dockerfile

@@ -1,25 +1,29 @@
-FROM node:12-alpine as base
+FROM node:14-alpine3.12 as base
 
 WORKDIR /app
-RUN apk add --no-cache tzdata eudev
+RUN apk add --no-cache tzdata eudev tini
 
 COPY package.json ./
 
-# Dependencies
-FROM base as dependencies
+# Dependencies and build
+FROM base as dependencies_and_build
 
-COPY npm-shrinkwrap.json ./
+COPY npm-shrinkwrap.json tsconfig.json index.js ./
+COPY lib ./lib
 
-RUN apk add --no-cache --virtual .buildtools make gcc g++ python linux-headers git && \
-    npm ci --production && \
+RUN apk add --no-cache --virtual .buildtools make gcc g++ python3 linux-headers git && \
+    npm ci --no-audit --no-optional --no-update-notifier && \
+    npm run build && \
+    rm -rf node_modules && \
+    npm ci --production --no-audit --no-optional --no-update-notifier && \
     apk del .buildtools
 
 # Release
 FROM base as release
 
-COPY --from=dependencies /app/node_modules ./node_modules
-COPY lib ./lib
-COPY index.js data/configuration.yaml ./
+COPY --from=dependencies_and_build /app/node_modules ./node_modules
+COPY --from=dependencies_and_build /app/dist ./dist
+COPY LICENSE index.js data/configuration.yaml ./
 
 COPY docker/docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
@@ -27,7 +31,7 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 RUN mkdir /app/data
 
 ARG COMMIT
-RUN echo "{\"hash\": \"$COMMIT\"}" > .hash.json
+RUN echo "$COMMIT" > dist/.hash
 
 ENTRYPOINT ["docker-entrypoint.sh"]
-CMD ["npm", "start"]
+CMD [ "/sbin/tini", "--", "node", "index.js"]

docker/README.md

@@ -1 +0,0 @@
-qemu-aarch64-static and qemu-arm-static are taken from https://github.com/resin-io/qemu/releases/tag/v2.9.0%2Bresin1

index.js

@@ -1,38 +1,117 @@
+require('core-js/features/object/from-entries');
+require('core-js/features/array/flat');
 const semver = require('semver');
 const engines = require('./package.json').engines;
+const indexJsRestart = 'indexjs.restart';
+const fs = require('fs');
+const path = require('path');
+const {exec} = require('child_process');
+const rimraf = require('rimraf');
+require('source-map-support').install();
 
-const version = engines.node;
-if (!semver.satisfies(process.version, version)) {
-    console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
-}
-
-// Validate settings
-const settings = require('./lib/util/settings');
-const errors = settings.validate();
-if (errors.length > 0) {
-    console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
-    console.log('            READ THIS CAREFULLY\n');
-    console.log(`Refusing to start because configuration is not valid, found the following errors:`);
-    for (const error of errors) {
-        console.log(`- ${error}`);
-    }
-    console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`);
-    console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
-    process.exit(1);
-}
-
-const Controller = require('./lib/controller');
-const controller = new Controller();
-controller.start();
-
-process.on('SIGINT', handleQuit);
-process.on('SIGTERM', handleQuit);
-
+let controller;
 let stopping = false;
 
-function handleQuit() {
-    if (!stopping) {
-        stopping = true;
-        controller.stop();
+const hashFile = path.join(__dirname, 'dist', '.hash');
+
+async function restart() {
+    await stop(indexJsRestart);
+    await start();
+}
+
+async function exit(code, reason) {
+    if (reason !== indexJsRestart) {
+        process.exit(code);
     }
 }
+
+async function currentHash() {
+    const git = require('git-last-commit');
+    return new Promise((resolve) => {
+        git.getLastCommit((err, commit) => {
+            if (err) resolve('unknown');
+            else resolve(commit.shortHash);
+        });
+    });
+}
+
+async function writeHash() {
+    const hash = await currentHash();
+    fs.writeFileSync(hashFile, hash);
+}
+
+async function build(reason) {
+    return new Promise((resolve, reject) => {
+        process.stdout.write(`Building Zigbee2MQTT... (${reason})`);
+        rimraf.sync('dist');
+        exec('npm run build', {cwd: __dirname}, async (err, stdout, stderr) => {
+            if (err) {
+                process.stdout.write(', failed\n');
+                reject(err);
+            } else {
+                process.stdout.write(', finished\n');
+                resolve();
+            }
+        });
+    });
+}
+
+async function checkDist() {
+    if (!fs.existsSync(hashFile)) {
+        await build('initial build');
+    }
+
+    const distHash = fs.readFileSync(hashFile, 'utf-8');
+    const hash = await currentHash();
+    if (hash !== 'unknown' && distHash !== hash) {
+        await build('hash changed');
+    }
+}
+
+async function start() {
+    await checkDist();
+
+    const version = engines.node;
+    if (!semver.satisfies(process.version, version)) {
+        console.log(`\t\tZigbee2MQTT requires node version ${version}, you are running ${process.version}!\n`); // eslint-disable-line
+    }
+
+    // Validate settings
+    const settings = require('./dist/util/settings');
+    settings.reRead();
+    const errors = settings.validate();
+    if (errors.length > 0) {
+        console.log(`\n\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!`);
+        console.log('            READ THIS CAREFULLY\n');
+        console.log(`Refusing to start because configuration is not valid, found the following errors:`);
+        for (const error of errors) {
+            console.log(`- ${error}`);
+        }
+        console.log(`\nIf you don't know how to solve this, read https://www.zigbee2mqtt.io/information/configuration.html`); // eslint-disable-line
+        console.log(`\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\n`);
+        exit(1);
+    }
+
+    const Controller = require('./dist/controller');
+    controller = new Controller(restart, exit);
+    await controller.start();
+}
+
+async function stop(reason=null) {
+    await controller.stop(reason);
+}
+
+async function handleQuit() {
+    if (!stopping && controller) {
+        stopping = true;
+        await stop();
+    }
+}
+
+if (process.argv.length === 3 && process.argv[2] === 'writehash') {
+    writeHash();
+} else {
+    process.on('SIGINT', handleQuit);
+    process.on('SIGTERM', handleQuit);
+    start();
+}

lib/controller.js

@@ -1,385 +0,0 @@
-const MQTT = require('./mqtt');
-const Zigbee = require('./zigbee');
-const EventBus = require('./eventBus');
-const State = require('./state');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const objectAssignDeep = require('object-assign-deep');
-const utils = require('./util/utils');
-const stringify = require('json-stable-stringify-without-jsonify');
-const fs = require('fs');
-const data = require('./util/data');
-const path = require('path');
-const assert = require('assert');
-
-// Extensions
-const ExtensionFrontend = require('./extension/frontend');
-const ExtensionPublish = require('./extension/publish');
-const ExtensionReceive = require('./extension/receive');
-const ExtensionNetworkMap = require('./extension/networkMap');
-const ExtensionSoftReset = require('./extension/legacy/softReset');
-const ExtensionHomeAssistant = require('./extension/homeassistant');
-const ExtensionConfigure = require('./extension/configure');
-const ExtensionDeviceGroupMembership = require('./extension/legacy/deviceGroupMembership');
-const ExtensionBridgeLegacy = require('./extension/legacy/bridgeLegacy');
-const ExtensionBridge = require('./extension/bridge');
-const ExtensionGroups = require('./extension/groups');
-const ExtensionAvailability = require('./extension/availability');
-const ExtensionBind = require('./extension/bind');
-const ExtensionReport = require('./extension/report');
-const ExtensionOnEvent = require('./extension/onEvent');
-const ExtensionOTAUpdate = require('./extension/otaUpdate');
-const ExtensionExternalConverters = require('./extension/externalConverters');
-
-const AllExtensions = [
-    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
-    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
-    ExtensionAvailability, ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
-    ExtensionExternalConverters,
-];
-
-class Controller {
-    constructor() {
-        this.zigbee = new Zigbee();
-        this.mqtt = new MQTT();
-        this.eventBus = new EventBus();
-        this.state = new State(this.eventBus);
-
-        this.publishEntityState = this.publishEntityState.bind(this);
-        this.enableDisableExtension = this.enableDisableExtension.bind(this);
-        this.onZigbeeAdapterDisconnected = this.onZigbeeAdapterDisconnected.bind(this);
-
-        // Initialize extensions.
-        const args = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus];
-        this.extensions = [
-            new ExtensionPublish(...args),
-            new ExtensionReceive(...args),
-            new ExtensionDeviceGroupMembership(...args),
-            new ExtensionConfigure(...args),
-            new ExtensionNetworkMap(...args),
-            new ExtensionGroups(...args),
-            new ExtensionBind(...args),
-            new ExtensionOnEvent(...args),
-            new ExtensionOTAUpdate(...args),
-            new ExtensionReport(...args),
-        ];
-
-        if (settings.get().experimental.new_api) {
-            this.extensions.push(new ExtensionBridge(...args, this.enableDisableExtension));
-        }
-
-        if (settings.get().frontend) {
-            this.extensions.push(new ExtensionFrontend(...args));
-        }
-
-        if (settings.get().advanced.legacy_api) {
-            this.extensions.push(new ExtensionBridgeLegacy(...args));
-        }
-
-        if (settings.get().homeassistant) {
-            this.extensions.push(new ExtensionHomeAssistant(...args));
-        }
-
-        /* istanbul ignore next */
-        if (settings.get().advanced.soft_reset_timeout !== 0) {
-            this.extensions.push(new ExtensionSoftReset(...args));
-        }
-
-        if (settings.get().advanced.availability_timeout) {
-            this.extensions.push(new ExtensionAvailability(...args));
-        }
-        if (settings.get().external_converters.length) {
-            this.extensions.push(new ExtensionExternalConverters(...args));
-        }
-
-        const extensionPath = data.joinPath('extension');
-        if (fs.existsSync(extensionPath)) {
-            const extensions = fs.readdirSync(extensionPath).filter((f) => f.endsWith('.js'));
-            for (const extension of extensions) {
-                const Extension = require(path.join(extensionPath, extension.split('.')[0]));
-                this.extensions.push(new Extension(...args, settings, logger));
-            }
-        }
-    }
-
-    async start() {
-        this.state.start();
-
-        const info = await utils.getZigbee2mqttVersion();
-        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
-
-        // Start zigbee
-        try {
-            await this.zigbee.start();
-            this.callExtensionMethod('onZigbeeStarted', []);
-            this.zigbee.on('event', this.onZigbeeEvent.bind(this));
-            this.zigbee.on('adapterDisconnected', this.onZigbeeAdapterDisconnected);
-        } catch (error) {
-            logger.error('Failed to start zigbee');
-            logger.error('Exiting...');
-            logger.error(error.stack);
-            process.exit(1);
-        }
-
-        // Log zigbee clients on startup
-        const devices = this.zigbee.getClients();
-        logger.info(`Currently ${devices.length} devices are joined:`);
-        for (const device of devices) {
-            const entity = this.zigbee.resolveEntity(device);
-            const model = entity.definition ?
-                `${entity.definition.model} - ${entity.definition.vendor} ${entity.definition.description}` :
-                'Not supported';
-            logger.info(`${entity.name} (${entity.device.ieeeAddr}): ${model} (${entity.device.type})`);
-        }
-
-        // Enable zigbee join.
-        if (settings.get().permit_join) {
-            logger.warn('`permit_join` set to  `true` in configuration.yaml.');
-            logger.warn('Allowing new devices to join.');
-            logger.warn('Set `permit_join` to `false` once you joined all devices.');
-        }
-
-        try {
-            await this.zigbee.permitJoin(settings.get().permit_join);
-        } catch (error) {
-            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
-        }
-
-        // MQTT
-        this.mqtt.on('message', this.onMQTTMessage.bind(this));
-        await this.mqtt.connect();
-        this.mqtt.publish('bridge/state', 'online', {retain: true, qos: 0});
-
-        // Send all cached states.
-        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
-            for (const device of this.zigbee.getClients()) {
-                if (this.state.exists(device.ieeeAddr)) {
-                    this.publishEntityState(device.ieeeAddr, this.state.get(device.ieeeAddr));
-                }
-            }
-        }
-
-        // Add devices which are in database but not in configuration to configuration
-        for (const device of this.zigbee.getClients()) {
-            if (!settings.getDevice(device.ieeeAddr)) {
-                settings.addDevice(device.ieeeAddr);
-            }
-        }
-
-        // Call extensions
-        await this.callExtensionMethod('onMQTTConnected', []);
-    }
-
-    async enableDisableExtension(enable, name) {
-        if (!enable) {
-            const extension = this.extensions.find((e) => e.constructor.name === name);
-            if (extension) {
-                await this.callExtensionMethod('stop', [], [extension]);
-                this.extensions.splice(this.extensions.indexOf(extension), 1);
-            }
-        } else {
-            const Extension = AllExtensions.find((e) => e.name === name);
-            assert(Extension, `Extension '${name}' does not exist`);
-            const extension = new Extension(this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus);
-            this.extensions.push(extension);
-            this.callExtensionMethod('onZigbeeStarted', [], [extension]);
-            this.callExtensionMethod('onMQTTConnected', [], [extension]);
-        }
-    }
-
-    async stop() {
-        // Call extensions
-        await this.callExtensionMethod('stop', []);
-
-        // Wrap-up
-        this.state.stop();
-        await this.mqtt.disconnect();
-
-        try {
-            await this.zigbee.stop();
-            process.exit(0);
-        } catch (error) {
-            logger.error('Failed to stop zigbee');
-            process.exit(1);
-        }
-    }
-
-    async onZigbeeAdapterDisconnected() {
-        logger.error('Adapter disconnected, stopping');
-        await this.stop();
-    }
-
-    async onZigbeeEvent(type, data) {
-        const resolvedEntity = this.zigbee.resolveEntity(data.device || data.ieeeAddr);
-        if (data.device && !settings.getDevice(data.device.ieeeAddr) && data.device.type !== 'Coordinator') {
-            // Only deviceLeave doesn't have a device (not interesting to add to settings)
-            resolvedEntity.settings = {...settings.get().device_options, ...settings.addDevice(data.device.ieeeAddr)};
-        }
-
-        const name = resolvedEntity && resolvedEntity.settings ? resolvedEntity.settings.friendlyName : null;
-
-        if (type === 'message') {
-            logger.debug(
-                `Received Zigbee message from '${name}', type '${data.type}', cluster '${data.cluster}'` +
-                `, data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
-                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``),
-            );
-        } else if (type === 'deviceJoined') {
-            logger.info(`Device '${name}' joined`);
-        } else if (type === 'deviceInterview') {
-            if (data.status === 'successful') {
-                logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
-
-                if (resolvedEntity.definition) {
-                    const {vendor, description, model} = resolvedEntity.definition;
-                    logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
-                } else {
-                    logger.warn(
-                        `Device '${name}' with Zigbee model '${data.device.modelID}' is NOT supported, ` +
-                        `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`,
-                    );
-                }
-            } else if (data.status === 'failed') {
-                logger.error(`Failed to interview '${name}', device has not successfully been paired`);
-            } else {
-                /* istanbul ignore else */
-                if (data.status === 'started') {
-                    logger.info(`Starting interview of '${name}'`);
-                }
-            }
-        } else if (type === 'deviceAnnounce') {
-            logger.debug(`Device '${name}' announced itself`);
-        } else {
-            /* istanbul ignore else */
-            if (type === 'deviceLeave') {
-                logger.warn(`Device '${name || data.ieeeAddr}' left the network`);
-            }
-        }
-
-        // Call extensions
-        this.callExtensionMethod('onZigbeeEvent', [type, data, resolvedEntity]);
-    }
-
-    onMQTTMessage(payload) {
-        const {topic, message} = payload;
-        logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
-
-        // Call extensions
-        this.callExtensionMethod('onMQTTMessage', [topic, message]);
-    }
-
-    async publishEntityState(IDorName, payload, stateChangeReason=null) {
-        const resolvedEntity = this.zigbee.resolveEntity(IDorName);
-        if (!resolvedEntity || !resolvedEntity.settings) {
-            logger.error(`'${IDorName}' does not exist, skipping publish`);
-            return;
-        }
-
-        let messagePayload = {...payload};
-        const currentState = this.state.exists(resolvedEntity.settings.ID) ?
-            this.state.get(resolvedEntity.settings.ID) : {};
-        const newState = objectAssignDeep.noMutate(currentState, payload);
-
-        // Update state cache with new state.
-        this.state.set(resolvedEntity.settings.ID, newState, stateChangeReason);
-
-        if (settings.get().advanced.cache_state) {
-            // Add cached state to payload
-            messagePayload = newState;
-        }
-
-        const options = {
-            retain: utils.getObjectProperty(resolvedEntity.settings, 'retain', false),
-            qos: utils.getObjectProperty(resolvedEntity.settings, 'qos', 0),
-        };
-
-        const retention = utils.getObjectProperty(resolvedEntity.settings, 'retention', false);
-        if (retention !== false) {
-            options.properties = {messageExpiryInterval: retention};
-        }
-
-        const isDevice = resolvedEntity.type === 'device';
-        if (isDevice && settings.get().mqtt.include_device_information) {
-            const attributes = [
-                'ieeeAddr', 'networkAddress', 'type', 'manufacturerID', 'manufacturerName', 'powerSource',
-                'applicationVersion', 'stackVersion', 'zclVersion', 'hardwareVersion', 'dateCode', 'softwareBuildID',
-            ];
-
-            messagePayload.device = {
-                friendlyName: resolvedEntity.name,
-                model: resolvedEntity.definition ? resolvedEntity.definition.model : 'unknown',
-            };
-
-            attributes.forEach((a) => messagePayload.device[a] = resolvedEntity.device[a]);
-        }
-
-        const lastSeen = settings.get().advanced.last_seen;
-        if (isDevice && lastSeen !== 'disable' && resolvedEntity.device.lastSeen) {
-            messagePayload.last_seen = utils.formatDate(resolvedEntity.device.lastSeen, lastSeen);
-        }
-
-        // filter mqtt message attributes
-        if (resolvedEntity.settings.filtered_attributes) {
-            resolvedEntity.settings.filtered_attributes.forEach((a) => delete messagePayload[a]);
-        }
-
-        if (Object.entries(messagePayload).length) {
-            const output = settings.get().experimental.output;
-            if (output === 'attribute_and_json' || output === 'json') {
-                await this.mqtt.publish(resolvedEntity.name, stringify(messagePayload), options);
-            }
-
-            if (output === 'attribute_and_json' || output === 'attribute') {
-                await this.iteratePayloadAttributeOutput(`${resolvedEntity.name}/`, messagePayload, options);
-            }
-        }
-
-        this.eventBus.emit('publishEntityState', {payload: messagePayload, entity: resolvedEntity, stateChangeReason});
-    }
-
-    async iteratePayloadAttributeOutput(topicRoot, payload, options) {
-        for (const [key, value] of Object.entries(payload)) {
-            let subPayload = value;
-            let message = null;
-
-            // Special cases
-            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
-                subPayload = [subPayload.r, subPayload.g, subPayload.b];
-            }
-
-            // Check Array first, since it is also an Object
-            if (subPayload === null || subPayload === undefined) {
-                message = '';
-            } else if (Array.isArray(subPayload)) {
-                message = subPayload.map((x) => `${x}`).join(',');
-            } else if (typeof subPayload === 'object') {
-                this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
-            } else {
-                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
-            }
-
-            if (message !== null) {
-                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
-            }
-        }
-    }
-
-    async callExtensionMethod(method, parameters, extensions=null) {
-        for (const extension of extensions || this.extensions) {
-            if (extension[method]) {
-                try {
-                    await extension[method](...parameters);
-                } catch (error) {
-                    /* istanbul ignore next */
-                    logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
-                    /* istanbul ignore next */
-                    if (process.env.JEST_WORKER_ID !== undefined) {
-                        throw error;
-                    }
-                }
-            }
-        }
-    }
-}
-
-module.exports = Controller;

lib/controller.ts

@@ -0,0 +1,321 @@
+import MQTT from './mqtt';
+import Zigbee from './zigbee';
+import EventBus from './eventBus';
+import State from './state';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import utils from './util/utils';
+import stringify from 'json-stable-stringify-without-jsonify';
+import assert from 'assert';
+import bind from 'bind-decorator';
+
+// Extensions
+import ExtensionFrontend from './extension/frontend';
+import ExtensionPublish from './extension/publish';
+import ExtensionReceive from './extension/receive';
+import ExtensionNetworkMap from './extension/networkMap';
+import ExtensionSoftReset from './extension/legacy/softReset';
+import ExtensionHomeAssistant from './extension/homeassistant';
+import ExtensionConfigure from './extension/configure';
+import ExtensionDeviceGroupMembership from './extension/legacy/deviceGroupMembership';
+import ExtensionBridgeLegacy from './extension/legacy/bridgeLegacy';
+import ExtensionBridge from './extension/bridge';
+import ExtensionGroups from './extension/groups';
+import ExtensionAvailability from './extension/availability';
+import ExtensionBind from './extension/bind';
+import ExtensionReport from './extension/legacy/report';
+import ExtensionOnEvent from './extension/onEvent';
+import ExtensionOTAUpdate from './extension/otaUpdate';
+import ExtensionExternalConverters from './extension/externalConverters';
+import ExtensionExternalExtension from './extension/externalExtension';
+
+const AllExtensions = [
+    ExtensionPublish, ExtensionReceive, ExtensionNetworkMap, ExtensionSoftReset, ExtensionHomeAssistant,
+    ExtensionConfigure, ExtensionDeviceGroupMembership, ExtensionBridgeLegacy, ExtensionBridge, ExtensionGroups,
+    ExtensionBind, ExtensionReport, ExtensionOnEvent, ExtensionOTAUpdate,
+    ExtensionExternalConverters, ExtensionFrontend, ExtensionExternalExtension, ExtensionAvailability,
+];
+
+type ExtensionArgs = [Zigbee, MQTT, State, PublishEntityState, EventBus,
+    (enable: boolean, name: string) => Promise<void>, () => void, (extension: Extension) => void];
+
+class Controller {
+    private eventBus: EventBus;
+    private zigbee: Zigbee;
+    private state: State;
+    private mqtt: MQTT;
+    private restartCallback: () => void;
+    private exitCallback: (code: number) => void;
+    private extensions: Extension[];
+    private extensionArgs: ExtensionArgs;
+
+    constructor(restartCallback: () => void, exitCallback: (code: number) => void) {
+        this.eventBus = new EventBus( /* istanbul ignore next */ (error) => {
+            logger.error(`Error: ${error.message}`);
+            logger.debug(error.stack);
+        });
+        this.zigbee = new Zigbee(this.eventBus);
+        this.mqtt = new MQTT(this.eventBus);
+        this.state = new State(this.eventBus);
+        this.restartCallback = restartCallback;
+        this.exitCallback = exitCallback;
+
+        // Initialize extensions.
+        this.extensionArgs = [this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus,
+            this.enableDisableExtension, this.restartCallback, this.addExtension];
+
+        this.extensions = [
+            new ExtensionBridge(...this.extensionArgs),
+            new ExtensionPublish(...this.extensionArgs),
+            new ExtensionReceive(...this.extensionArgs),
+            new ExtensionDeviceGroupMembership(...this.extensionArgs),
+            new ExtensionConfigure(...this.extensionArgs),
+            new ExtensionNetworkMap(...this.extensionArgs),
+            new ExtensionGroups(...this.extensionArgs),
+            new ExtensionBind(...this.extensionArgs),
+            new ExtensionOnEvent(...this.extensionArgs),
+            new ExtensionOTAUpdate(...this.extensionArgs),
+            new ExtensionReport(...this.extensionArgs),
+            new ExtensionExternalExtension(...this.extensionArgs),
+            new ExtensionAvailability(...this.extensionArgs),
+            settings.get().frontend && new ExtensionFrontend(...this.extensionArgs),
+            settings.get().advanced.legacy_api && new ExtensionBridgeLegacy(...this.extensionArgs),
+            settings.get().external_converters.length && new ExtensionExternalConverters(...this.extensionArgs),
+            settings.get().homeassistant && new ExtensionHomeAssistant(...this.extensionArgs),
+            /* istanbul ignore next */
+            settings.get().advanced.soft_reset_timeout !== 0 && new ExtensionSoftReset(...this.extensionArgs),
+        ].filter((n) => n);
+    }
+
+    async start(): Promise<void> {
+        this.state.start();
+        logger.logOutput();
+
+        const info = await utils.getZigbee2MQTTVersion();
+        logger.info(`Starting Zigbee2MQTT version ${info.version} (commit #${info.commitHash})`);
+
+        // Start zigbee
+        let startResult;
+        try {
+            startResult = await this.zigbee.start();
+            this.eventBus.onAdapterDisconnected(this, this.onZigbeeAdapterDisconnected);
+        } catch (error) {
+            logger.error('Failed to start zigbee');
+            logger.error('Check https://www.zigbee2mqtt.io/information/FAQ.html#help-zigbee2mqtt-fails-to-start for possible solutions'); /* eslint-disable-line max-len */
+            logger.error('Exiting...');
+            logger.error(error.stack);
+            this.exitCallback(1);
+        }
+
+        // Disable some legacy options on new network creation
+        if (startResult === 'reset') {
+            settings.set(['advanced', 'homeassistant_legacy_entity_attributes'], false);
+            settings.set(['advanced', 'legacy_api'], false);
+            settings.set(['device_options', 'legacy'], false);
+            this.enableDisableExtension(false, 'BridgeLegacy');
+        }
+
+        // Log zigbee clients on startup
+        const devices = this.zigbee.devices(false);
+        logger.info(`Currently ${devices.length} devices are joined:`);
+        for (const device of devices) {
+            const model = device.definition ?
+                `${device.definition.model} - ${device.definition.vendor} ${device.definition.description}` :
+                'Not supported';
+            logger.info(`${device.name} (${device.ieeeAddr}): ${model} (${device.zh.type})`);
+        }
+
+        // Enable zigbee join
+        try {
+            if (settings.get().permit_join) {
+                logger.warn('`permit_join` set to  `true` in configuration.yaml.');
+                logger.warn('Allowing new devices to join.');
+                logger.warn('Set `permit_join` to `false` once you joined all devices.');
+            }
+
+            await this.zigbee.permitJoin(settings.get().permit_join);
+        } catch (error) {
+            logger.error(`Failed to set permit join to ${settings.get().permit_join}`);
+        }
+
+        // MQTT
+        try {
+            await this.mqtt.connect();
+        } catch (error) {
+            logger.error(`MQTT failed to connect: ${error.message}`);
+            logger.error('Exiting...');
+            await this.zigbee.stop();
+            this.exitCallback(1);
+        }
+
+        // Call extensions
+        await this.callExtensions('start', [...this.extensions]);
+
+        // Send all cached states.
+        if (settings.get().advanced.cache_state_send_on_startup && settings.get().advanced.cache_state) {
+            for (const device of devices) {
+                if (this.state.exists(device)) {
+                    this.publishEntityState(device, this.state.get(device));
+                }
+            }
+        }
+
+        if (settings.get().advanced.last_seen && settings.get().advanced.last_seen !== 'disable') {
+            this.eventBus.onLastSeenChanged(this, (data) =>
+                this.publishEntityState(data.device, {}, 'lastSeenChanged'));
+        }
+    }
+
+    @bind async enableDisableExtension(enable: boolean, name: string): Promise<void> {
+        if (!enable) {
+            const extension = this.extensions.find((e) => e.constructor.name === name);
+            if (extension) {
+                await this.callExtensions('stop', [extension]);
+                this.extensions.splice(this.extensions.indexOf(extension), 1);
+            }
+        } else {
+            const Extension = AllExtensions.find((e) => e.name === name);
+            assert(Extension, `Extension '${name}' does not exist`);
+            const extension = new Extension(...this.extensionArgs);
+            this.extensions.push(extension);
+            await this.callExtensions('start', [extension]);
+        }
+    }
+
+    @bind async addExtension(extension: Extension): Promise<void> {
+        this.extensions.push(extension);
+        await this.callExtensions('start', [extension]);
+    }
+
+    async stop(): Promise<void> {
+        // Call extensions
+        await this.callExtensions('stop', this.extensions);
+        this.eventBus.removeListeners(this);
+
+        // Wrap-up
+        this.state.stop();
+        await this.mqtt.disconnect();
+
+        try {
+            await this.zigbee.stop();
+            logger.info('Stopped Zigbee2MQTT');
+            this.exitCallback(0);
+        } catch (error) {
+            logger.error('Failed to stop Zigbee2MQTT');
+            this.exitCallback(1);
+        }
+    }
+
+    @bind async onZigbeeAdapterDisconnected(): Promise<void> {
+        logger.error('Adapter disconnected, stopping');
+        await this.stop();
+    }
+
+    @bind async publishEntityState(entity: Group | Device, payload: KeyValue,
+        stateChangeReason?: StateChangeReason): Promise<void> {
+        let message = {...payload};
+
+        // Update state cache with new state.
+        const newState = this.state.set(entity, payload, stateChangeReason);
+
+        if (settings.get().advanced.cache_state) {
+            // Add cached state to payload
+            message = newState;
+        }
+
+        const options: MQTTOptions = {
+            retain: utils.getObjectProperty(entity.settings, 'retain', false) as boolean,
+            qos: utils.getObjectProperty(entity.settings, 'qos', 0) as 0 | 1 | 2,
+        };
+
+        const retention = utils.getObjectProperty(entity.settings, 'retention', false);
+        if (retention !== false) {
+            options.properties = {messageExpiryInterval: retention as number};
+        }
+
+        if (entity.isDevice() && settings.get().mqtt.include_device_information) {
+            message.device = {
+                friendlyName: entity.name, model: entity.definition ? entity.definition.model : 'unknown',
+                ieeeAddr: entity.ieeeAddr, networkAddress: entity.zh.networkAddress, type: entity.zh.type,
+                manufacturerID: entity.zh.manufacturerID, manufacturerName: entity.zh.manufacturerName,
+                powerSource: entity.zh.powerSource, applicationVersion: entity.zh.applicationVersion,
+                stackVersion: entity.zh.stackVersion, zclVersion: entity.zh.zclVersion,
+                hardwareVersion: entity.zh.hardwareVersion, dateCode: entity.zh.dateCode,
+                softwareBuildID: entity.zh.softwareBuildID,
+            };
+        }
+
+        // Add lastseen
+        const lastSeen = settings.get().advanced.last_seen;
+        if (entity.isDevice() && lastSeen !== 'disable' && entity.zh.lastSeen) {
+            message.last_seen = utils.formatDate(entity.zh.lastSeen, lastSeen);
+        }
+
+        // Add device linkquality.
+        if (entity.isDevice() && entity.zh.linkquality !== undefined) {
+            message.linkquality = entity.zh.linkquality;
+        }
+
+        // filter mqtt message attributes
+        if (entity.settings.filtered_attributes) {
+            entity.settings.filtered_attributes.forEach((a) => delete message[a]);
+        }
+
+        for (const extension of this.extensions) {
+            extension.adjustMessageBeforePublish?.(entity, message);
+        }
+
+        if (Object.entries(message).length) {
+            const output = settings.get().experimental.output;
+            if (output === 'attribute_and_json' || output === 'json') {
+                await this.mqtt.publish(entity.name, stringify(message), options);
+            }
+
+            if (output === 'attribute_and_json' || output === 'attribute') {
+                await this.iteratePayloadAttributeOutput(`${entity.name}/`, message, options);
+            }
+        }
+
+        this.eventBus.emitPublishEntityState({entity, message, stateChangeReason});
+    }
+
+    async iteratePayloadAttributeOutput(topicRoot: string, payload: KeyValue, options: MQTTOptions): Promise<void> {
+        for (const [key, value] of Object.entries(payload)) {
+            let subPayload = value;
+            let message = null;
+
+            // Special cases
+            if (key === 'color' && utils.objectHasProperties(subPayload, ['r', 'g', 'b'])) {
+                subPayload = [subPayload.r, subPayload.g, subPayload.b];
+            }
+
+            // Check Array first, since it is also an Object
+            if (subPayload === null || subPayload === undefined) {
+                message = '';
+            } else if (Array.isArray(subPayload)) {
+                message = subPayload.map((x) => `${x}`).join(',');
+            } else if (typeof subPayload === 'object') {
+                await this.iteratePayloadAttributeOutput(`${topicRoot}${key}-`, subPayload, options);
+            } else {
+                message = typeof subPayload === 'string' ? subPayload : stringify(subPayload);
+            }
+
+            if (message !== null) {
+                await this.mqtt.publish(`${topicRoot}${key}`, message, options);
+            }
+        }
+    }
+
+    private async callExtensions(method: 'start' | 'stop', extensions: Extension[]): Promise<void> {
+        for (const extension of extensions) {
+            try {
+                await extension[method]?.();
+            } catch (error) {
+                /* istanbul ignore next */
+                logger.error(`Failed to call '${extension.constructor.name}' '${method}' (${error.stack})`);
+            }
+        }
+    }
+}
+
+module.exports = Controller;

lib/eventBus.js

@@ -1,43 +0,0 @@
-const events = require('events');
-const assert = require('assert');
-
-const allowedEvents = [
-    'deviceRemoved', // Device has been removed
-    'deviceRenamed', // Device has been renamed
-    'groupRenamed', // Group has been renamed
-    'publishEntityState', // Entity state will be published
-    'stateChange', // Entity changes its state
-    'groupMembersChanged', // Members of a group has been changed
-    'reportingDisabled', // Reporting is disabled for a device
-    'devicesChanged', // Devices changed
-];
-
-class EventBus extends events.EventEmitter {
-    constructor() {
-        super();
-        this.callbackByExtension = {};
-    }
-
-    emit(event, data) {
-        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        super.emit(event, data);
-    }
-
-    on(event, callback, extension=null) {
-        assert(allowedEvents.includes(event), `Event '${event}' not supported`);
-        if (extension) {
-            if (!this.callbackByExtension[extension]) this.callbackByExtension[extension] = [];
-            this.callbackByExtension[extension].push({event, callback});
-        }
-
-        super.on(event, callback);
-    }
-
-    removeListenersExtension(extension) {
-        for (const entry of this.callbackByExtension[extension] || []) {
-            super.removeListener(entry.event, entry.callback);
-        }
-    }
-}
-
-module.exports = EventBus;

lib/eventBus.ts

@@ -0,0 +1,160 @@
+import events from 'events';
+events.captureRejections = true;
+
+// eslint-disable-next-line
+type ListenerKey = object;
+
+export default class EventBus {
+    private callbacksByExtension: { [s: string]: { event: string, callback: (...args: unknown[]) => void }[] } = {};
+    private emitter = new events.EventEmitter();
+
+    constructor(onError: (error: Error) => void) {
+        this.emitter.setMaxListeners(100);
+        this.emitter.on('error', onError);
+    }
+
+    public emitAdapterDisconnected(): void {
+        this.emitter.emit('adapterDisconnected');
+    }
+    public onAdapterDisconnected(key: ListenerKey, callback: () => void): void {
+        this.on('adapterDisconnected', callback, key);
+    }
+
+    public emitPermitJoinChanged(data: eventdata.PermitJoinChanged): void {
+        this.emitter.emit('permitJoinChanged', data);
+    }
+    public onPermitJoinChanged(key: ListenerKey, callback: (data: eventdata.PermitJoinChanged) => void): void {
+        this.on('permitJoinChanged', callback, key);
+    }
+
+    public emitDeviceRenamed(data: eventdata.DeviceRenamed): void {
+        this.emitter.emit('deviceRenamed', data);
+    }
+    public onDeviceRenamed(key: ListenerKey, callback: (data: eventdata.DeviceRenamed) => void): void {
+        this.on('deviceRenamed', callback, key);
+    }
+
+    public emitDeviceRemoved(data: eventdata.DeviceRemoved): void {
+        this.emitter.emit('deviceRemoved', data);
+    }
+    public onDeviceRemoved(key: ListenerKey, callback: (data: eventdata.DeviceRemoved) => void): void {
+        this.on('deviceRemoved', callback, key);
+    }
+
+    public emitLastSeenChanged(data: eventdata.LastSeenChanged): void {
+        this.emitter.emit('lastSeenChanged', data);
+    }
+    public onLastSeenChanged(key: ListenerKey, callback: (data: eventdata.LastSeenChanged) => void): void {
+        this.on('lastSeenChanged', callback, key);
+    }
+
+    public emitDeviceNetworkAddressChanged(data: eventdata.DeviceNetworkAddressChanged): void {
+        this.emitter.emit('deviceNetworkAddressChanged', data);
+    }
+    public onDeviceNetworkAddressChanged(
+        key: ListenerKey, callback: (data: eventdata.DeviceNetworkAddressChanged) => void): void {
+        this.on('deviceNetworkAddressChanged', callback, key);
+    }
+
+    public emitDeviceAnnounce(data: eventdata.DeviceAnnounce): void {
+        this.emitter.emit('deviceAnnounce', data);
+    }
+    public onDeviceAnnounce(key: ListenerKey, callback: (data: eventdata.DeviceAnnounce) => void): void {
+        this.on('deviceAnnounce', callback, key);
+    }
+
+    public emitDeviceInterview(data: eventdata.DeviceInterview): void {
+        this.emitter.emit('deviceInterview', data);
+    }
+    public onDeviceInterview(key: ListenerKey, callback: (data: eventdata.DeviceInterview) => void): void {
+        this.on('deviceInterview', callback, key);
+    }
+
+    public emitDeviceJoined(data: eventdata.DeviceJoined): void {
+        this.emitter.emit('deviceJoined', data);
+    }
+    public onDeviceJoined(key: ListenerKey, callback: (data: eventdata.DeviceJoined) => void): void {
+        this.on('deviceJoined', callback, key);
+    }
+
+    public emitDeviceLeave(data: eventdata.DeviceLeave): void {
+        this.emitter.emit('deviceLeave', data);
+    }
+    public onDeviceLeave(key: ListenerKey, callback: (data: eventdata.DeviceLeave) => void): void {
+        this.on('deviceLeave', callback, key);
+    }
+
+    public emitDeviceMessage(data: eventdata.DeviceMessage): void {
+        this.emitter.emit('deviceMessage', data);
+    }
+    public onDeviceMessage(key: ListenerKey, callback: (data: eventdata.DeviceMessage) => void): void {
+        this.on('deviceMessage', callback, key);
+    }
+
+    public emitMQTTMessage(data: eventdata.MQTTMessage): void {
+        this.emitter.emit('mqttMessage', data);
+    }
+    public onMQTTMessage(key: ListenerKey, callback: (data: eventdata.MQTTMessage) => void): void {
+        this.on('mqttMessage', callback, key);
+    }
+
+    public emitMQTTMessagePublished(data: eventdata.MQTTMessagePublished): void {
+        this.emitter.emit('mqttMessagePublished', data);
+    }
+    public onMQTTMessagePublished(key: ListenerKey, callback: (data: eventdata.MQTTMessagePublished) => void): void {
+        this.on('mqttMessagePublished', callback, key);
+    }
+
+    public emitPublishEntityState(data: eventdata.PublishEntityState): void {
+        this.emitter.emit('publishEntityState', data);
+    }
+    public onPublishEntityState(key: ListenerKey, callback: (data: eventdata.PublishEntityState) => void): void {
+        this.on('publishEntityState', callback, key);
+    }
+
+    public emitGroupMembersChanged(data: eventdata.GroupMembersChanged): void {
+        this.emitter.emit('groupMembersChanged', data);
+    }
+    public onGroupMembersChanged(key: ListenerKey, callback: (data: eventdata.GroupMembersChanged) => void): void {
+        this.on('groupMembersChanged', callback, key);
+    }
+
+    public emitDevicesChanged(): void {
+        this.emitter.emit('devicesChanged');
+    }
+    public onDevicesChanged(key: ListenerKey, callback: () => void): void {
+        this.on('devicesChanged', callback, key);
+    }
+
+    public emitScenesChanged(): void {
+        this.emitter.emit('scenesChanged');
+    }
+    public onScenesChanged(key: ListenerKey, callback: () => void): void {
+        this.on('scenesChanged', callback, key);
+    }
+
+    public emitReportingDisabled(data: eventdata.ReportingDisabled): void {
+        this.emitter.emit('reportingDisabled', data);
+    }
+    public onReportingDisabled(key: ListenerKey, callback: (data: eventdata.ReportingDisabled) => void): void {
+        this.on('reportingDisabled', callback, key);
+    }
+
+    public emitStateChange(data: eventdata.StateChange): void {
+        this.emitter.emit('stateChange', data);
+    }
+    public onStateChange(key: ListenerKey, callback: (data: eventdata.StateChange) => void): void {
+        this.on('stateChange', callback, key);
+    }
+
+    private on(event: string, callback: (...args: unknown[]) => void, key: ListenerKey): void {
+        if (!this.callbacksByExtension[key.constructor.name]) this.callbacksByExtension[key.constructor.name] = [];
+        this.callbacksByExtension[key.constructor.name].push({event, callback});
+        this.emitter.on(event, callback);
+    }
+
+    public removeListeners(key: ListenerKey): void {
+        this.callbacksByExtension[key.constructor.name]?.forEach(
+            (e) => this.emitter.removeListener(e.event, e.callback));
+    }
+}

lib/extension/availability.js

@@ -1,206 +0,0 @@
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const Extension = require('./extension');
-
-// Pingable end devices, some end devices should be pinged
-// e.g. E11-G13 https://github.com/Koenkk/zigbee2mqtt/issues/775#issuecomment-453683846
-const pingableEndDevices = [
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'E11-G13'),
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === '53170161'),
-];
-
-const Hours25 = 1000 * 60 * 60 * 25;
-
-/**
- * This extensions pings devices to check if they are online.
- */
-class Availability extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        this.availability_timeout = settings.get().advanced.availability_timeout;
-        this.timers = {};
-        this.state = {};
-
-        this.eventBus.on('deviceRemoved', (data) => this.onDeviceRemoved(data.resolvedEntity), this.constructor.name);
-
-        this.blocklist = settings.get().advanced.availability_blocklist
-            .concat(settings.get().advanced.availability_blacklist)
-            .map((e) => settings.getEntity(e).ID);
-
-        this.passlist = settings.get().advanced.availability_passlist
-            .concat(settings.get().advanced.availability_whitelist)
-            .map((e) => settings.getEntity(e).ID);
-    }
-
-    onDeviceRemoved(resolvedEntity) {
-        this.mqtt.publish(`${resolvedEntity.name}/availability`, null, {retain: true, qos: 0});
-        delete this.state[resolvedEntity.device.ieeeAddr];
-        clearTimeout(this.timers[resolvedEntity.device.ieeeAddr]);
-    }
-
-    inPasslistOrNotInBlocklist(device) {
-        const ieeeAddr = device.ieeeAddr;
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings.friendlyName;
-
-        // Passlist is not empty and device is in it, enable availability
-        if (this.passlist.length > 0) {
-            return this.passlist.includes(ieeeAddr) || (name && this.passlist.includes(name));
-        }
-
-        // Device is on blocklist, disable availability
-        if (this.blocklist.includes(ieeeAddr) || (name && this.blocklist.includes(name))) {
-            return false;
-        }
-
-        return true;
-    }
-
-    isPingable(device) {
-        if (pingableEndDevices.find((d) => d.hasOwnProperty('zigbeeModel') && d.zigbeeModel.includes(device.modelID))) {
-            return true;
-        }
-
-        // Device is a mains powered router
-        return device.type === 'Router' && device.powerSource !== 'Battery';
-    }
-
-    onMQTTConnected() {
-        for (const device of this.zigbee.getClients()) {
-            // Mark all devices as online on start
-            const ieeeAddr = device.ieeeAddr;
-            this.publishAvailability(device, this.state.hasOwnProperty(ieeeAddr) ? this.state[ieeeAddr] : true, true);
-
-            if (this.inPasslistOrNotInBlocklist(device)) {
-                if (this.isPingable(device)) {
-                    this.setTimerPingable(device);
-                } else {
-                    this.timers[ieeeAddr] = setInterval(() => {
-                        this.handleIntervalNotPingable(device);
-                    }, utils.secondsToMilliseconds(300));
-                }
-            }
-        }
-    }
-
-    async handleIntervalPingable(device) {
-        // When a device is already unavailable, log the ping failed on 'debug' instead of 'error'.
-        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!resolvedEntity) {
-            logger.debug(`Stop pinging '${device.ieeeAddr}', device is not known anymore`);
-            return;
-        }
-
-        const level = this.state.hasOwnProperty(device.ieeeAddr) && !this.state[device.ieeeAddr] ? 'debug' : 'error';
-        try {
-            await device.ping();
-            this.publishAvailability(device, true);
-            logger.debug(`Successfully pinged '${resolvedEntity.name}'`);
-        } catch (error) {
-            this.publishAvailability(device, false);
-            logger[level](`Failed to ping '${resolvedEntity.name}'`);
-        } finally {
-            this.setTimerPingable(device);
-        }
-    }
-
-    async handleIntervalNotPingable(device) {
-        const resolvedEntity = this.zigbee.resolveEntity(device.ieeeAddr);
-        if (!resolvedEntity || !device.lastSeen) {
-            return;
-        }
-
-        const ago = Date.now() - resolvedEntity.device.lastSeen;
-        logger.debug(`Non-pingable device '${resolvedEntity.name}' was last seen '${ago / 1000}' seconds ago.`);
-
-        if (ago > Hours25) {
-            this.publishAvailability(device, false);
-        }
-    }
-
-    setTimerPingable(device) {
-        clearTimeout(this.timers[device.ieeeAddr]);
-        this.timers[device.ieeeAddr] = setTimeout(async () => {
-            await this.handleIntervalPingable(device);
-        }, utils.secondsToMilliseconds(this.availability_timeout));
-    }
-
-    async stop() {
-        super.stop();
-        for (const timer of Object.values(this.timers)) {
-            clearTimeout(timer);
-        }
-
-        this.zigbee.getClients().forEach((device) => this.publishAvailability(device, false));
-    }
-
-    async onReconnect(device) {
-        const resolvedEntity = this.zigbee.resolveEntity(device);
-        if (resolvedEntity && resolvedEntity.definition) {
-            const used = [];
-            try {
-                for (const key of ['state', 'brightness', 'color', 'color_temp']) {
-                    const converter = resolvedEntity.definition.toZigbee.find((tz) => tz.key.includes(key));
-                    if (converter && !used.includes(converter)) {
-                        await converter.convertGet(device.endpoints[0], key, {});
-                        used.push(converter);
-                    }
-                }
-            } catch (error) {
-                logger.error(`Failed to read state of '${resolvedEntity.name}' after reconnect`);
-            }
-        }
-    }
-
-    publishAvailability(device, available, force=false) {
-        const ieeeAddr = device.ieeeAddr;
-        if (this.state.hasOwnProperty(ieeeAddr) && !this.state[ieeeAddr] && available) {
-            this.onReconnect(device);
-        }
-
-        const deviceSettings = settings.getDevice(ieeeAddr);
-        const name = deviceSettings ? deviceSettings.friendlyName : ieeeAddr;
-        const topic = `${name}/availability`;
-        const payload = available ? 'online' : 'offline';
-        if (this.state[ieeeAddr] !== available || force) {
-            this.state[ieeeAddr] = available;
-            this.mqtt.publish(topic, payload, {retain: true, qos: 0});
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        const device = data.device;
-        if (!device) {
-            return;
-        }
-
-        if (this.inPasslistOrNotInBlocklist(device)) {
-            this.publishAvailability(data.device, true);
-
-            if (this.isPingable(device)) {
-                // When a zigbee message from a device is received we know the device is still alive.
-                // => reset the timer.
-                this.setTimerPingable(device);
-
-                const online = this.state.hasOwnProperty(device.ieeeAddr) && this.state[device.ieeeAddr];
-                if (online && type === 'deviceAnnounce' && !utils.isIkeaTradfriDevice(device)) {
-                    /**
-                     * In case the device is powered off AND on within the availability timeout,
-                     * zigbee2qmtt does not detect the device as offline (device is still marked online).
-                     * When a device is turned on again the state could be out of sync.
-                     * https://github.com/Koenkk/zigbee2mqtt/issues/1383#issuecomment-489412168
-                     * deviceAnnounce is typically send when a device comes online.
-                     *
-                     * This isn't needed for TRADFRI devices as they already send the state themself.
-                     */
-                    this.onReconnect(device);
-                }
-            }
-        }
-    }
-}
-
-module.exports = Availability;

lib/extension/availability.ts

@@ -0,0 +1,188 @@
+import Extension from './extension';
+import logger from '../util/logger';
+import utils from '../util/utils';
+import * as settings from '../util/settings';
+import debounce from 'debounce';
+import bind from 'bind-decorator';
+
+export default class Availability extends Extension {
+    private timers: {[s: string]: NodeJS.Timeout} = {};
+    private availabilityCache: {[s: string]: boolean} = {};
+    private retrieveStateDebouncers: {[s: string]: () => void} = {};
+    private pingQueue: Device[] = [];
+    private pingQueueExecuting = false;
+
+    private getTimeout(device: Device): number {
+        if (typeof device.settings.availability === 'object' && device.settings.availability?.timeout != null) {
+            return utils.minutes(device.settings.availability.timeout);
+        }
+
+        const key = this.isActiveDevice(device) ? 'active' : 'passive';
+        const availabilitySettings = settings.get().availability;
+        if (typeof availabilitySettings === 'object' && availabilitySettings[key]?.timeout != null) {
+            return utils.minutes(availabilitySettings[key]?.timeout);
+        }
+
+        return key === 'active' ? utils.minutes(10) : utils.hours(25);
+    }
+
+    private isActiveDevice(device: Device): boolean {
+        return (device.zh.type === 'Router' && device.zh.powerSource !== 'Battery') ||
+            device.zh.powerSource === 'Mains (single phase)';
+    }
+
+    private isAvailable(device: Device): boolean {
+        const ago = Date.now() - device.zh.lastSeen;
+        return ago < this.getTimeout(device);
+    }
+
+    private resetTimer(device: Device): void {
+        clearTimeout(this.timers[device.ieeeAddr]);
+
+        // If the timer triggers, the device is not avaiable anymore otherwise resetTimer already have been called
+        if (this.isActiveDevice(device)) {
+            // If device did not check in, ping it, if that fails it will be marked as offline
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.addToPingQueue(device), this.getTimeout(device) + utils.seconds(1));
+        } else {
+            this.timers[device.ieeeAddr] = setTimeout(
+                () => this.publishAvailability(device, true), this.getTimeout(device) + utils.seconds(1));
+        }
+    }
+
+    private addToPingQueue(device: Device): void {
+        this.pingQueue.push(device);
+        this.pingQueueExecuteNext();
+    }
+
+    private removeFromPingQueue(device: Device): void {
+        const index = this.pingQueue.findIndex((d) => d.ieeeAddr === device.ieeeAddr);
+        index != -1 && this.pingQueue.splice(index, 1);
+    }
+
+    private async pingQueueExecuteNext(): Promise<void> {
+        if (this.pingQueue.length === 0 || this.pingQueueExecuting) return;
+        this.pingQueueExecuting = true;
+
+        const device = this.pingQueue[0];
+        let pingedSuccessfully = false;
+        const available = this.availabilityCache[device.ieeeAddr] || this.isAvailable(device);
+        const attempts = available ? 2 : 1;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                // Enable recovery if device is marked as available and first ping fails.
+                const disableRecovery = !(i == 1 && available);
+                await device.zh.ping(disableRecovery);
+                pingedSuccessfully = true;
+                logger.debug(`Succesfully pinged '${device.name}' (attempt ${i + 1}/${attempts})`);
+                break;
+            } catch (error) {
+                logger.error(`Failed to ping '${device.name}' (attempt ${i + 1}/${attempts}, ${error.message})`);
+                // Try again in 3 seconds.
+                const lastAttempt = i - 1 === attempts;
+                !lastAttempt && await utils.sleep(3);
+            }
+        }
+
+        this.publishAvailability(device, !pingedSuccessfully);
+        this.resetTimer(device);
+        this.removeFromPingQueue(device);
+
+        // Sleep 2 seconds before executing next ping
+        await utils.sleep(2);
+        this.pingQueueExecuting = false;
+        this.pingQueueExecuteNext();
+    }
+
+    override async start(): Promise<void> {
+        logger.warn('Using experimental new availability feature');
+
+        this.eventBus.onDeviceRenamed(this, (data: eventdata.DeviceRenamed) =>
+            this.publishAvailability(data.device, false, true));
+        this.eventBus.onDeviceLeave(this, (data: eventdata.DeviceLeave) => clearTimeout(this.timers[data.ieeeAddr]));
+        this.eventBus.onDeviceAnnounce(this, (data: eventdata.DeviceAnnounce) => this.retrieveState(data.device));
+        this.eventBus.onLastSeenChanged(this, this.onLastSeenChanged);
+
+        for (const device of this.zigbee.devices(false)) {
+            if (utils.isAvailabilityEnabledForDevice(device, settings.get())) {
+                // Publish initial availablility
+                this.publishAvailability(device, true);
+
+                this.resetTimer(device);
+
+                // If an active device is initially unavailable, ping it.
+                if (this.isActiveDevice(device) && !this.isAvailable(device)) {
+                    this.addToPingQueue(device);
+                }
+            }
+        }
+    }
+
+    private publishAvailability(device: Device, logLastSeen: boolean, forcePublish=false): void {
+        if (logLastSeen) {
+            const ago = Date.now() - device.zh.lastSeen;
+            if (this.isActiveDevice(device)) {
+                logger.debug(`Active device '${device.name}' was last seen ` +
+                    `'${(ago / utils.minutes(1)).toFixed(2)}' minutes ago.`);
+            } else {
+                logger.debug(
+                    `Passive device '${device.name}' was last seen '${(ago / utils.hours(1)).toFixed(2)}' hours ago.`);
+            }
+        }
+
+        const available = this.isAvailable(device);
+        if (!forcePublish && this.availabilityCache[device.ieeeAddr] == available) {
+            return;
+        }
+
+        if (device.ieeeAddr in this.availabilityCache && available &&
+            this.availabilityCache[device.ieeeAddr] === false) {
+            logger.debug(`Device '${device.name}' reconnected`);
+            this.retrieveState(device);
+        }
+
+        const topic = `${device.name}/availability`;
+        const payload = available ? 'online' : 'offline';
+        this.availabilityCache[device.ieeeAddr] = available;
+        this.mqtt.publish(topic, payload, {retain: true, qos: 0});
+    }
+
+    @bind private onLastSeenChanged(data: eventdata.LastSeenChanged): void {
+        if (utils.isAvailabilityEnabledForDevice(data.device, settings.get())) {
+            // Remove from ping queue, not necessary anymore since we know the device is online.
+            this.removeFromPingQueue(data.device);
+            this.resetTimer(data.device);
+            this.publishAvailability(data.device, false);
+        }
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        Object.values(this.timers).forEach((t) => clearTimeout(t));
+    }
+
+    private retrieveState(device: Device): void {
+        /**
+         * Retrieve state of a device in a debounced manner, this function is called on a 'deviceAnnounce' which a
+         * device can send multiple times after each other.
+         */
+        if (device.definition && !device.zh.interviewing && !this.retrieveStateDebouncers[device.ieeeAddr]) {
+            this.retrieveStateDebouncers[device.ieeeAddr] = debounce(async () => {
+                try {
+                    logger.debug(`Retrieving state of '${device.name}' after reconnect`);
+                    // Color and color temperature converters do both, only needs to be called once.
+                    const keySet = [['state'], ['brightness'], ['color', 'color_temp']];
+                    for (const keys of keySet) {
+                        const converter = device.definition.toZigbee.find((c) => c.key.find((k) => keys.includes(k)));
+                        await converter?.convertGet?.(device.endpoint(), keys[0],
+                            {message: this.state.get(device) || {}, mapped: device.definition});
+                    }
+                } catch (error) {
+                    logger.error(`Failed to read state of '${device.name}' after reconnect (${error.message})`);
+                }
+            }, utils.seconds(2));
+        }
+
+        this.retrieveStateDebouncers[device.ieeeAddr]?.();
+    }
+}

lib/extension/bind.js

@@ -1,166 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering'];
-
-// See zigbee-herdsman-converters devices.js
-const defaultBindGroup = {type: 'group_number', ID: 901};
-
-class Bind extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/bind/#`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/unbind/#`);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/bind`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/unbind`);
-        }
-    }
-
-    parseMQTTMessage(topic, message) {
-        let type = null;
-        let sourceKey = null;
-        let targetKey = null;
-        let clusters = null;
-
-        if (this.legacyApi && topic.match(legacyTopicRegex)) {
-            topic = topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
-            type = topic.split('/')[0];
-            sourceKey = topic.replace(`${type}/`, '');
-            targetKey = message;
-        } else if (settings.get().experimental.new_api && topic.match(topicRegex)) {
-            type = topic.endsWith('unbind') ? 'unbind' : 'bind';
-            message = JSON.parse(message);
-            sourceKey = message.from;
-            targetKey = message.to;
-            clusters = message.clusters;
-        }
-
-        return {type, sourceKey, targetKey, clusters};
-    }
-
-    async onMQTTMessage(topic, message) {
-        const {type, sourceKey, targetKey, clusters} = this.parseMQTTMessage(topic, message);
-        if (!type) return null;
-
-        let error = null;
-        const source = this.zigbee.resolveEntity(sourceKey);
-        const target = targetKey === 'default_bind_group' ? defaultBindGroup : this.zigbee.resolveEntity(targetKey);
-        const responseData = {from: sourceKey, to: targetKey};
-
-        if (!source || source.type !== 'device') {
-            error = `Source device '${sourceKey}' does not exist`;
-        } else if (!target) {
-            error = `Target device or group '${targetKey}' does not exist`;
-        } else {
-            const sourceName = source.settings.friendlyName;
-            const targetName = targetKey === 'default_bind_group' ? targetKey : target.settings.friendlyName;
-            const successfulClusters = [];
-            const failedClusters = [];
-            const attemptedClusters = [];
-
-            // Find which clusters are supported by both the source and target.
-            // Groups are assumed to support all clusters.
-            for (const cluster of clusterCandidates) {
-                if (clusters && !clusters.includes(cluster)) continue;
-
-                const targetValid = target.type === 'group' || target.type === 'group_number' ||
-                    target.device.type === 'Coordinator' || target.endpoint.supportsInputCluster(cluster);
-
-                if (source.endpoint.supportsOutputCluster(cluster) && targetValid) {
-                    logger.debug(`${type}ing cluster '${cluster}' from '${sourceName}' to '${targetName}'`);
-                    attemptedClusters.push(cluster);
-
-                    try {
-                        let bindTarget = null;
-                        if (target.type === 'group') bindTarget = target.group;
-                        else if (target.type === 'group_number') bindTarget = target.ID;
-                        else bindTarget = target.endpoint;
-
-                        if (type === 'bind') {
-                            await source.endpoint.bind(cluster, bindTarget);
-                        } else {
-                            await source.endpoint.unbind(cluster, bindTarget);
-                        }
-
-                        successfulClusters.push(cluster);
-                        logger.info(
-                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
-                            `'${sourceName}' to '${targetName}'`,
-                        );
-
-                        /* istanbul ignore else */
-                        if (settings.get().advanced.legacy_api) {
-                            this.mqtt.publish(
-                                'bridge/log',
-                                stringify({type: `device_${type}`,
-                                    message: {from: sourceName, to: targetName, cluster}}),
-                            );
-                        }
-                    } catch (error) {
-                        failedClusters.push(cluster);
-                        logger.error(
-                            `Failed to ${type} cluster '${cluster}' from '${sourceName}' to ` +
-                            `'${targetName}' (${error})`,
-                        );
-
-                        /* istanbul ignore else */
-                        if (settings.get().advanced.legacy_api) {
-                            this.mqtt.publish(
-                                'bridge/log',
-                                stringify({type: `device_${type}_failed`,
-                                    message: {from: sourceName, to: targetName, cluster}}),
-                            );
-                        }
-                    }
-                }
-            }
-
-            if (attemptedClusters.length === 0) {
-                logger.error(`Nothing to ${type} from '${sourceName}' to '${targetName}'`);
-                error = `Nothing to ${type}`;
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    this.mqtt.publish(
-                        'bridge/log',
-                        stringify({type: `device_${type}_failed`, message: {from: sourceName, to: targetName}}),
-                    );
-                }
-            } else if (failedClusters.length === attemptedClusters.length) {
-                error = `Failed to ${type}`;
-            }
-
-            responseData[`clusters`] = successfulClusters;
-            responseData[`failed`] = failedClusters;
-        }
-
-        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
-        if (!triggeredViaLegacyApi) {
-            const response = utils.getResponse(message, responseData, error);
-            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
-        }
-
-        if (error) {
-            logger.error(error);
-        } else {
-            this.eventBus.emit(`devicesChanged`);
-        }
-    }
-}
-
-module.exports = Bind;

lib/extension/bind.ts

@@ -0,0 +1,478 @@
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import utils from '../util/utils';
+import Extension from './extension';
+import stringify from 'json-stable-stringify-without-jsonify';
+import debounce from 'debounce';
+import * as zigbeeHersdman from 'zigbee-herdsman/dist';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+import Group from '../model/group';
+
+const legacyApi = settings.get().advanced.legacy_api;
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/(bind|unbind)/.+$`);
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/(bind|unbind)`);
+const clusterCandidates = ['genScenes', 'genOnOff', 'genLevelCtrl', 'lightingColorCtrl', 'closuresWindowCovering',
+    'hvacThermostat', 'msTemperatureMeasurement'];
+
+// See zigbee-herdsman-converters
+const defaultBindGroup = {type: 'group_number', ID: 901, name: 'default_bind_group'};
+
+const defaultReportConfiguration = {
+    minimumReportInterval: 5, maximumReportInterval: 3600, reportableChange: 1,
+};
+
+const getColorCapabilities = async (endpoint: zh.Endpoint): Promise<{colorTemperature: boolean, colorXY: boolean}> => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') == null) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') as number;
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const reportClusters: {[s: string]:
+    {attribute: string, minimumReportInterval: number, maximumReportInterval: number, reportableChange: number
+        condition?: (endpoint: zh.Endpoint) => Promise<boolean>}[]} =
+{
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultReportConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultReportConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultReportConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultReportConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultReportConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultReportConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultReportConfiguration},
+    ],
+};
+
+type PollOnMessage = {
+    cluster: {[s: string]: {type: string, data: KeyValue}[]}
+    read: {cluster: string, attributes: string[], attributesForEndpoint?: (endpoint: zh.Endpoint) => Promise<string[]>}
+    manufacturerIDs: number[]
+}[];
+
+const pollOnMessage: PollOnMessage = [
+    {
+        // On messages that have the cluster and type of below
+        cluster: {
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 2}},
+                {type: 'commandHueNotification', data: {button: 3}},
+            ],
+            genLevelCtrl: [
+                {type: 'commandStep', data: {}},
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandStop', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMove', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+        },
+        // Read the following attributes
+        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
+        // When the bound devices/members of group have the following manufacturerIDs
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genLevelCtrl: [
+                {type: 'commandStepWithOnOff', data: {}},
+                {type: 'commandMoveWithOnOff', data: {}},
+                {type: 'commandStopWithOnOff', data: {}},
+                {type: 'commandMoveToLevelWithOnOff', data: {}},
+            ],
+            genOnOff: [
+                {type: 'commandOn', data: {}},
+                {type: 'commandOff', data: {}},
+                {type: 'commandOffWithEffect', data: {}},
+                {type: 'commandToggle', data: {}},
+            ],
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+            manuSpecificPhilips: [
+                {type: 'commandHueNotification', data: {button: 1}},
+                {type: 'commandHueNotification', data: {button: 4}},
+            ],
+        },
+        read: {cluster: 'genOnOff', attributes: ['onOff']},
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+    {
+        cluster: {
+            genScenes: [
+                {type: 'commandRecall', data: {}},
+            ],
+        },
+        read: {
+            cluster: 'lightingColorCtrl',
+            attributes: [] as string[],
+            // Since not all devices support the same attributes they need to be calculated dynamically
+            // depending on the capabilities of the endpoint.
+            attributesForEndpoint: async (endpoint): Promise<string[]> => {
+                const supportedAttrs = await getColorCapabilities(endpoint);
+                const readAttrs: string[] = [];
+                supportedAttrs.colorXY && readAttrs.push('currentX', 'currentY');
+                supportedAttrs.colorTemperature && readAttrs.push('colorTemperature');
+                return readAttrs;
+            },
+        },
+        manufacturerIDs: [
+            zigbeeHersdman.Zcl.ManufacturerCode.Philips,
+            zigbeeHersdman.Zcl.ManufacturerCode.ATMEL,
+            zigbeeHersdman.Zcl.ManufacturerCode.GLEDOPTO_CO_LTD,
+            zigbeeHersdman.Zcl.ManufacturerCode.MUELLER_LICHT_INT,
+        ],
+    },
+];
+
+interface ParsedMQTTMessage {
+    type: 'bind' | 'unbind', sourceKey: string, targetKey: string, clusters: string[], skipDisableReporting: boolean
+}
+
+export default class Bind extends Extension {
+    private pollDebouncers: {[s: string]: () => void} = {};
+
+    override async start(): Promise<void> {
+        this.eventBus.onDeviceMessage(this, this.poll);
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+        this.eventBus.onGroupMembersChanged(this, this.onGroupMembersChanged);
+    }
+
+    private parseMQTTMessage(data: eventdata.MQTTMessage): ParsedMQTTMessage {
+        let type: 'bind' | 'unbind' = null;
+        let sourceKey = null;
+        let targetKey = null;
+        let clusters = null;
+        let skipDisableReporting = false;
+
+        if (legacyApi && data.topic.match(legacyTopicRegex)) {
+            const topic = data.topic.replace(`${settings.get().mqtt.base_topic}/bridge/`, '');
+            type = topic.split('/')[0] as 'bind' | 'unbind';
+            sourceKey = topic.replace(`${type}/`, '');
+            targetKey = data.message;
+        } else if (data.topic.match(topicRegex)) {
+            type = data.topic.endsWith('unbind') ? 'unbind' : 'bind';
+            const message = JSON.parse(data.message);
+            sourceKey = message.from;
+            targetKey = message.to;
+            clusters = message.clusters;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
+        }
+
+        return {type, sourceKey, targetKey, clusters, skipDisableReporting};
+    }
+
+    @bind private async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const {type, sourceKey, targetKey, clusters, skipDisableReporting} = this.parseMQTTMessage(data);
+        if (!type) return null;
+        const message = utils.parseJSON(data.message, data.message);
+
+        let error = null;
+        const parsedSource = utils.parseEntityID(sourceKey);
+        const parsedTarget = utils.parseEntityID(targetKey);
+        const source = this.zigbee.resolveEntity(parsedSource.ID);
+        const target = targetKey === 'default_bind_group' ?
+            defaultBindGroup : this.zigbee.resolveEntity(parsedTarget.ID);
+        const responseData: KeyValue = {from: sourceKey, to: targetKey};
+
+        if (!source || !(source instanceof Device)) {
+            error = `Source device '${sourceKey}' does not exist`;
+        } else if (!target) {
+            error = `Target device or group '${targetKey}' does not exist`;
+        } else {
+            const successfulClusters: string[] = [];
+            const failedClusters = [];
+            const attemptedClusters = [];
+
+            const bindSource: zh.Endpoint = source.endpoint(parsedSource.endpoint);
+            let bindTarget: number | zh.Group | zh.Endpoint = null;
+            if (target instanceof Device) bindTarget = target.endpoint(parsedTarget.endpoint);
+            else if (target instanceof Group) bindTarget = target.zh;
+            else bindTarget = Number(target.ID);
+
+            // Find which clusters are supported by both the source and target.
+            // Groups are assumed to support all clusters.
+            for (const cluster of clusterCandidates) {
+                if (clusters && !clusters.includes(cluster)) continue;
+                let matchingClusters = false;
+
+                const anyClusterValid = utils.isZHGroup(bindTarget) || typeof bindTarget === 'number' ||
+                    (target as Device).zh.type === 'Coordinator';
+
+                if (!anyClusterValid && utils.isEndpoint(bindTarget)) {
+                    matchingClusters = ((bindTarget.supportsInputCluster(cluster) &&
+                            bindSource.supportsOutputCluster(cluster)) ||
+                            (bindSource.supportsInputCluster(cluster) &&
+                            bindTarget.supportsOutputCluster(cluster)) );
+                }
+
+                const sourceValid = bindSource.supportsInputCluster(cluster) ||
+                bindSource.supportsOutputCluster(cluster);
+
+                if ( sourceValid && (anyClusterValid || matchingClusters)) {
+                    logger.debug(`${type}ing cluster '${cluster}' from '${source.name}' to '${target.name}'`);
+                    attemptedClusters.push(cluster);
+
+                    try {
+                        if (type === 'bind') {
+                            await bindSource.bind(cluster, bindTarget);
+                        } else {
+                            await bindSource.unbind(cluster, bindTarget);
+                        }
+
+                        successfulClusters.push(cluster);
+                        logger.info(
+                            `Successfully ${type === 'bind' ? 'bound' : 'unbound'} cluster '${cluster}' from ` +
+                            `'${source.name}' to '${target.name}'`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}`,
+                                    message: {from: source.name, to: target.name, cluster}}),
+                            );
+                        }
+                    } catch (error) {
+                        failedClusters.push(cluster);
+                        logger.error(
+                            `Failed to ${type} cluster '${cluster}' from '${source.name}' to ` +
+                            `'${target.name}' (${error})`,
+                        );
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_${type}_failed`,
+                                    message: {from: source.name, to: target.name, cluster}}),
+                            );
+                        }
+                    }
+                }
+            }
+
+            if (attemptedClusters.length === 0) {
+                logger.error(`Nothing to ${type} from '${source.name}' to '${target.name}'`);
+                error = `Nothing to ${type}`;
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_${type}_failed`, message: {from: source.name, to: target.name}}),
+                    );
+                }
+            } else if (failedClusters.length === attemptedClusters.length) {
+                error = `Failed to ${type}`;
+            }
+
+            responseData[`clusters`] = successfulClusters;
+            responseData[`failed`] = failedClusters;
+
+            if (successfulClusters.length !== 0) {
+                if (type === 'bind') {
+                    await this.setupReporting(bindSource.binds.filter((b) =>
+                        successfulClusters.includes(b.cluster.name) && b.target === bindTarget));
+                } else if ((typeof bindTarget !== 'number') && !skipDisableReporting) {
+                    await this.disableUnnecessaryReportings(bindTarget);
+                }
+            }
+        }
+
+        const triggeredViaLegacyApi = data.topic.match(legacyTopicRegex);
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/device/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emitDevicesChanged();
+        }
+    }
+
+    @bind async onGroupMembersChanged(data: eventdata.GroupMembersChanged): Promise<void> {
+        if (data.action === 'add') {
+            const bindsToGroup = this.zigbee.devices(false).map((c) => c.zh.endpoints)
+                .reduce((a, v) => a.concat(v)).map((e) => e.binds)
+                .reduce((a, v) => a.concat(v)).filter((b) => b.target === data.group.zh);
+            await this.setupReporting(bindsToGroup);
+        } else { // action === remove/remove_all
+            if (!data.skipDisableReporting) {
+                await this.disableUnnecessaryReportings(data.endpoint);
+            }
+        }
+    }
+
+    getSetupReportingEndpoints(bind: zh.Bind, coordinatorEp: zh.Endpoint): zh.Endpoint[] {
+        const endpoints = utils.isEndpoint(bind.target) ? [bind.target] : bind.target.members;
+        return endpoints.filter((e) => {
+            const supportsInputCluster = e.supportsInputCluster(bind.cluster.name);
+            const hasConfiguredReporting = !!e.configuredReportings.find((c) => c.cluster.name === bind.cluster.name);
+            const hasBind = !!e.binds.find((b) => b.cluster.name === bind.cluster.name && b.target === coordinatorEp);
+            return supportsInputCluster && !(hasBind && hasConfiguredReporting);
+        });
+    }
+
+    async setupReporting(binds: zh.Bind[]): Promise<void> {
+        const coordinatorEndpoint = this.zigbee.firstCoordinatorEndpoint();
+        for (const bind of binds.filter((b) => b.cluster.name in reportClusters)) {
+            for (const endpoint of this.getSetupReportingEndpoints(bind, coordinatorEndpoint)) {
+                const entity = `${this.zigbee.resolveEntity(endpoint.getDevice()).name}/${endpoint.ID}`;
+                try {
+                    await endpoint.bind(bind.cluster.name, coordinatorEndpoint);
+                    const items = [];
+                    for (const c of reportClusters[bind.cluster.name]) {
+                        /* istanbul ignore else */
+                        if (!c.condition || await c.condition(endpoint)) {
+                            const i = {...c};
+                            delete i.condition;
+                            items.push(i);
+                        }
+                    }
+
+                    await endpoint.configureReporting(bind.cluster.name, items);
+                    logger.info(`Succesfully setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                } catch (error) {
+                    logger.warn(`Failed to setup reporting for '${entity}' cluster '${bind.cluster.name}'`);
+                }
+            }
+        }
+
+        this.eventBus.emitDevicesChanged();
+    }
+
+    async disableUnnecessaryReportings(target: zh.Group | zh.Endpoint): Promise<void> {
+        const coordinator = this.zigbee.firstCoordinatorEndpoint();
+        const endpoints = utils.isEndpoint(target) ? [target] : target.members;
+        for (const endpoint of endpoints) {
+            const device = this.zigbee.resolveEntity(endpoint.getDevice()) as Device;
+            const entity = `${device.name}/${endpoint.ID}`;
+            const boundClusters = endpoint.binds.filter((b) => b.target === coordinator)
+                .map((b) => b.cluster.name);
+            const requiredClusters = this.zigbee.devices(false).map((c) => c.zh.endpoints)
+                .reduce((a, v) => a.concat(v))
+                .map((e) => e.binds).reduce((a, v) => a.concat(v)).filter((bind) => {
+                    if (utils.isEndpoint(bind.target)) {
+                        return bind.target === endpoint;
+                    } else {
+                        return bind.target.members.includes(endpoint);
+                    }
+                }).map((b) => b.cluster.name).filter((v, i, a) => a.indexOf(v) === i);
+
+            for (const cluster of boundClusters.filter((c) => !requiredClusters.includes(c) && c in reportClusters)) {
+                try {
+                    await endpoint.unbind(cluster, coordinator);
+                    const items = [];
+                    for (const item of reportClusters[cluster]) {
+                        /* istanbul ignore else */
+                        if (!item.condition || await item.condition(endpoint)) {
+                            const i = {...item};
+                            delete i.condition;
+                            items.push({...i, maximumReportInterval: 0xFFFF});
+                        }
+                    }
+
+                    await endpoint.configureReporting(cluster, items);
+                    logger.info(`Succesfully disabled reporting for '${entity}' cluster '${cluster}'`);
+                } catch (error) {
+                    logger.warn(`Failed to disable reporting for '${entity}' cluster '${cluster}'`);
+                }
+            }
+
+            this.eventBus.emitReportingDisabled({device});
+        }
+    }
+
+    @bind async poll(data: eventdata.DeviceMessage): Promise<void> {
+        /**
+         * This method poll bound endpoints and group members for state changes.
+         *
+         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
+         * Hue bulbs only report their on/off state.
+         * When dimming the bulb via the dimmer switch the state is therefore not reported.
+         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
+         */
+        const polls = pollOnMessage.filter((p) =>
+            p.cluster[data.cluster]?.find((c) => c.type === data.type && utils.equalsPartial(data.data, c.data)));
+
+        if (polls.length) {
+            const toPoll: Set<zh.Endpoint> = new Set();
+            // Add bound devices
+            for (const endpoint of data.device.zh.endpoints) {
+                for (const bind of endpoint.binds) {
+                    if (utils.isEndpoint(bind.target) && bind.target.getDevice().type !== 'Coordinator') {
+                        toPoll.add(bind.target);
+                    }
+                }
+            }
+
+            // If message is published to a group, add members of the group
+            const group = data.groupID && data.groupID !== 0 && this.zigbee.groupByID(data.groupID);
+            if (group) {
+                group.zh.members.forEach((m) => toPoll.add(m));
+            }
+
+            for (const endpoint of toPoll) {
+                for (const poll of polls) {
+                    if (!poll.manufacturerIDs.includes(endpoint.getDevice().manufacturerID) ||
+                        !endpoint.supportsInputCluster(poll.read.cluster)) {
+                        continue;
+                    }
+
+                    let readAttrs = poll.read.attributes;
+                    if (poll.read.attributesForEndpoint) {
+                        const attrsForEndpoint = await poll.read.attributesForEndpoint(endpoint);
+                        readAttrs = [...poll.read.attributes, ...attrsForEndpoint];
+                    }
+
+                    const key = `${endpoint.getDevice().ieeeAddr}_${endpoint.ID}_${pollOnMessage.indexOf(poll)}`;
+                    if (!this.pollDebouncers[key]) {
+                        this.pollDebouncers[key] = debounce(async () => {
+                            await endpoint.read(poll.read.cluster, readAttrs);
+                        }, 1000);
+                    }
+
+                    this.pollDebouncers[key]();
+                }
+            }
+        }
+    }
+}

lib/extension/bridge.js

@@ -1,512 +0,0 @@
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const Extension = require('./extension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const settings = require('../util/settings');
-const Transport = require('winston-transport');
-const stringify = require('json-stable-stringify-without-jsonify');
-const objectAssignDeep = require(`object-assign-deep`);
-
-const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
-
-class Bridge extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.enableDisableExtension = enableDisableExtension;
-        this.lastJoinedDeviceIeeeAddr = null;
-        this.setupMQTTLogging();
-
-        this.requestLookup = {
-            'device/options': this.deviceOptions.bind(this),
-            'device/remove': this.deviceRemove.bind(this),
-            'device/rename': this.deviceRename.bind(this),
-            'group/add': this.groupAdd.bind(this),
-            'group/options': this.groupOptions.bind(this),
-            'group/remove': this.groupRemove.bind(this),
-            'group/rename': this.groupRename.bind(this),
-            'permit_join': this.permitJoin.bind(this),
-            'config/last_seen': this.configLastSeen.bind(this),
-            'config/homeassistant': this.configHomeAssistant.bind(this),
-            'config/elapsed': this.configElapsed.bind(this),
-            'config/log_level': this.configLogLevel.bind(this),
-            'touchlink/factory_reset': this.touchlinkFactoryReset.bind(this),
-            'touchlink/identify': this.touchlinkIdentify.bind(this),
-            'touchlink/scan': this.touchlinkScan.bind(this),
-            'health_check': this.healthCheck.bind(this),
-        };
-    }
-
-    async onMQTTConnected() {
-        this.zigbee2mqttVersion = await utils.getZigbee2mqttVersion();
-        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
-
-        this.eventBus.on(`groupMembersChanged`, () => this.publishGroups());
-        this.eventBus.on(`devicesChanged`, () => this.publishDevices());
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/#`);
-        await this.publishInfo();
-        await this.publishDevices();
-        await this.publishGroups();
-    }
-
-    setupMQTTLogging() {
-        const mqtt = this.mqtt;
-        class EventTransport extends Transport {
-            log(info, callback) {
-                const payload = stringify({message: info.message, level: info.level});
-                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
-                callback();
-            }
-        }
-
-        logger.add(new EventTransport());
-    }
-
-    async onMQTTMessage(topic, message) {
-        const match = topic.match(requestRegex);
-        if (match && this.requestLookup[match[1].toLowerCase()]) {
-            message = utils.parseJSON(message, message);
-
-            try {
-                const response = await this.requestLookup[match[1].toLowerCase()](message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
-            } catch (error) {
-                logger.error(`Request '${topic}' failed with error: '${error.message}'`);
-                const response = utils.getResponse(message, {}, error.message);
-                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
-            }
-        }
-    }
-
-    async onZigbeeEvent(type, data, resolvedEntity) {
-        if (type === 'deviceJoined' && resolvedEntity) {
-            this.lastJoinedDeviceIeeeAddr = resolvedEntity.device.ieeeAddr;
-        }
-
-        if (['deviceJoined', 'deviceLeave', 'deviceInterview', 'deviceAnnounce'].includes(type)) {
-            let payload;
-            const ieeeAddress = data.device ? data.device.ieeeAddr : data.ieeeAddr;
-            if (type === 'deviceJoined') {
-                payload = {friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress};
-            } else if (type === 'deviceInterview') {
-                payload = {
-                    friendly_name: resolvedEntity.settings.friendlyName, status: data.status, ieee_address: ieeeAddress,
-                };
-                if (data.status === 'successful') {
-                    payload.supported = !!resolvedEntity.definition;
-                    payload.definition = resolvedEntity.definition ? {
-                        model: resolvedEntity.definition.model,
-                        vendor: resolvedEntity.definition.vendor,
-                        description: resolvedEntity.definition.description,
-                        supports: resolvedEntity.definition.supports,
-                    } : null;
-                }
-            } else if (type === 'deviceAnnounce') {
-                payload = {
-                    friendly_name: resolvedEntity.settings.friendlyName, ieee_address: ieeeAddress,
-                };
-            } else payload = {ieee_address: ieeeAddress}; // deviceLeave
-
-            await this.mqtt.publish(
-                'bridge/event',
-                stringify({type: utils.toSnakeCase(type), data: payload}),
-                {retain: false, qos: 0},
-            );
-        }
-
-        if ('deviceLeave' === type || ('deviceInterview' === type && data.status !== 'started')) {
-            await this.publishDevices();
-        }
-    }
-
-    /**
-     * Requests
-     */
-
-    async deviceOptions(message) {
-        return this.changeEntityOptions('device', message);
-    }
-
-    async groupOptions(message) {
-        return this.changeEntityOptions('group', message);
-    }
-
-    async deviceRemove(message) {
-        return this.removeEntity('device', message);
-    }
-
-    async groupRemove(message) {
-        return this.removeEntity('group', message);
-    }
-
-    async healthCheck(message) {
-        return utils.getResponse(message, {healthy: true}, null);
-    }
-
-    async groupAdd(message) {
-        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
-            throw new Error(`Invalid payload`);
-        }
-
-        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
-        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
-        const group = settings.addGroup(friendlyName, ID);
-        this.zigbee.createGroup(group.ID);
-        this.publishGroups();
-        return utils.getResponse(message, {friendly_name: group.friendlyName, id: group.ID}, null);
-    }
-
-    async deviceRename(message) {
-        return this.renameEntity('device', message);
-    }
-
-    async groupRename(message) {
-        return this.renameEntity('group', message);
-    }
-
-    async permitJoin(message) {
-        const value = this.getValue(message);
-        await this.zigbee.permitJoin(value);
-        await this.publishInfo();
-        return utils.getResponse(message, {value: value}, null);
-    }
-
-    configLastSeen(message) {
-        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
-        const value = this.getValue(message);
-        if (!allowed.includes(value)) {
-            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
-        }
-
-        settings.set(['advanced', 'last_seen'], message);
-        this.publishInfo();
-        return utils.getResponse(message, {value}, null);
-    }
-
-    configHomeAssistant(message) {
-        const allowed = [true, false];
-        const value = this.getValue(message);
-        if (!allowed.includes(value)) {
-            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
-        }
-
-        this.enableDisableExtension(value, 'HomeAssistant');
-        settings.set(['homeassistant'], value);
-        this.publishInfo();
-        return utils.getResponse(message, {value}, null);
-    }
-
-    configElapsed(message) {
-        const allowed = [true, false];
-        const value = this.getValue(message);
-        if (!allowed.includes(value)) {
-            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
-        }
-
-        settings.set(['advanced', 'elapsed'], value);
-        this.publishInfo();
-        return utils.getResponse(message, {value}, null);
-    }
-
-    configLogLevel(message) {
-        const allowed = ['error', 'warn', 'info', 'debug'];
-        const value = this.getValue(message);
-        if (!allowed.includes(value)) {
-            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
-        }
-
-        logger.setLevel(value);
-        this.publishInfo();
-        return utils.getResponse(message, {value}, null);
-    }
-
-    async touchlinkIdentify(message) {
-        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
-            !message.hasOwnProperty('channel')) {
-            throw new Error('Invalid payload');
-        }
-
-        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
-        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
-        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
-    }
-
-    async touchlinkFactoryReset(message) {
-        let result = false;
-        const payload = {};
-        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
-            message.hasOwnProperty('channel')) {
-            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
-            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
-            payload.ieee_address = message.ieee_address;
-            payload.channel = message.channel;
-        } else {
-            logger.info('Start Touchlink factory reset of first found device');
-            result = await this.zigbee.touchlinkFactoryResetFirst();
-        }
-
-        if (result) {
-            logger.info('Successfully factory reset device through Touchlink');
-            return utils.getResponse(message, payload, null);
-        } else {
-            logger.error('Failed to factory reset device through Touchlink');
-            throw new Error('Failed to factory reset device through Touchlink');
-        }
-    }
-
-    async touchlinkScan(message) {
-        logger.info('Start Touchlink scan');
-        const result = await this.zigbee.touchlinkScan();
-        const found = result.map((r) => {
-            return {ieee_address: r.ieeeAddr, channel: r.channel};
-        });
-        logger.info('Finished Touchlink scan');
-        return utils.getResponse(message, {found}, null);
-    }
-
-    /**
-     * Utils
-     */
-
-    getValue(message) {
-        if (typeof message === 'object') {
-            if (!message.hasOwnProperty('value')) {
-                throw new Error('No value given');
-            }
-
-            return message.value;
-        } else {
-            return message;
-        }
-    }
-
-    changeEntityOptions(entityType, message) {
-        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
-            throw new Error(`Invalid payload`);
-        }
-
-        const ID = message.id;
-        const entity = this.getEntity(entityType, ID);
-        settings.changeEntityOptions(ID, message.options);
-        const cleanup = (o) => {
-            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
-            return o;
-        };
-        const oldOptions = cleanup(entity.settings);
-        const newOptions = cleanup(settings.getEntity(ID));
-        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
-    }
-
-    renameEntity(entityType, message) {
-        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
-        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
-            !message.hasOwnProperty('to')) {
-            throw new Error(`Invalid payload`);
-        }
-
-        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
-            throw new Error('No device has joined since start');
-        }
-
-        const validationErrors = utils.validateFriendlyName(message.to);
-        if (validationErrors.length !== 0) {
-            throw new Error(validationErrors[0]);
-        }
-
-        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
-        const to = message.to;
-        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
-            message.homeassistant_rename : false;
-        const entity = this.getEntity(entityType, from);
-
-        settings.changeFriendlyName(from, to);
-
-        // Clear retained messages
-        this.mqtt.publish(entity.name, '', {retain: true});
-
-        if (entity.type === 'device') {
-            this.publishDevices();
-            this.eventBus.emit(`deviceRenamed`, {device: entity.device, homeAssisantRename});
-        } else {
-            this.publishGroups();
-            this.eventBus.emit(`groupRenamed`, {group: entity.group, homeAssisantRename});
-        }
-
-        return utils.getResponse(
-            message,
-            {from: entity.settings.friendlyName, to, homeassistant_rename: homeAssisantRename},
-            null,
-        );
-    }
-
-    async removeEntity(entityType, message) {
-        const ID = typeof message === 'object' ? message.id : message.trim();
-        const entity = this.getEntity(entityType, ID);
-
-        let block = false;
-        let force = false;
-        let blockForceLog = '';
-
-        if (entityType === 'device' && typeof message === 'object') {
-            block = !!message.block;
-            force = !!message.force;
-            blockForceLog = ` (block: ${block}, force: ${force})`;
-        } else if (entityType === 'group' && typeof message === 'object') {
-            force = !!message.force;
-            blockForceLog = ` (force: ${force})`;
-        }
-
-        try {
-            logger.info(`Removing ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
-            if (entity.type === 'device') {
-                if (block) {
-                    settings.blockDevice(entity.settings.ID);
-                }
-
-                if (force) {
-                    await entity.device.removeFromDatabase();
-                } else {
-                    await entity.device.removeFromNetwork();
-                }
-            } else {
-                if (force) {
-                    await entity.group.removeFromDatabase();
-                } else {
-                    await entity.group.removeFromNetwork();
-                }
-            }
-
-            // Fire event
-            if (entity.type === 'device') {
-                this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
-            }
-
-            // Remove from configuration.yaml
-            if (entity.type === 'device') {
-                settings.removeDevice(entity.settings.ID);
-            } else {
-                settings.removeGroup(entity.settings.ID);
-            }
-
-            // Remove from state
-            this.state.remove(entity.settings.ID);
-
-            // Clear any retained messages
-            this.mqtt.publish(entity.name, '', {retain: true});
-
-            logger.info(`Successfully removed ${entity.type} '${entity.settings.friendlyName}'${blockForceLog}`);
-
-            if (entity.type === 'device') {
-                this.publishDevices();
-                return utils.getResponse(message, {id: ID, block, force}, null);
-            } else {
-                this.publishGroups();
-                return utils.getResponse(message, {id: ID, force: force}, null);
-            }
-        } catch (error) {
-            throw new Error(
-                `Failed to remove ${entity.type} '${entity.settings.friendlyName}'${blockForceLog} (${error})`,
-            );
-        }
-    }
-
-    getEntity(type, ID) {
-        const entity = this.zigbee.resolveEntity(ID);
-        if (!entity || entity.type !== type) {
-            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
-        }
-        return entity;
-    }
-
-    async publishInfo() {
-        const config = objectAssignDeep.noMutate({}, settings.get());
-        delete config.advanced.network_key;
-        const payload = {
-            version: this.zigbee2mqttVersion.version,
-            commit: this.zigbee2mqttVersion.commitHash,
-            coordinator: this.coordinatorVersion,
-            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
-            log_level: logger.getLevel(),
-            permit_join: await this.zigbee.getPermitJoin(),
-            config,
-        };
-
-        await this.mqtt.publish('bridge/info', stringify(payload), {retain: true, qos: 0});
-    }
-
-    async publishDevices() {
-        const devices = this.zigbee.getDevices().map((device) => {
-            const definition = zigbeeHerdsmanConverters.findByDevice(device);
-            const resolved = this.zigbee.resolveEntity(device);
-            const definitionPayload = definition ? {
-                model: definition.model,
-                vendor: definition.vendor,
-                description: definition.description,
-                supports: definition.supports,
-            } : null;
-
-            const endpoints = {};
-            for (const endpoint of device.endpoints) {
-                const data = {
-                    bindings: [],
-                    clusters: {
-                        input: endpoint.getInputClusters().map((c) => c.name),
-                        output: endpoint.getOutputClusters().map((c) => c.name),
-                    },
-                };
-
-                for (const bind of endpoint.binds) {
-                    let target;
-
-                    if (bind.target.constructor.name === 'Endpoint') {
-                        target = {
-                            type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID,
-                        };
-                    } else {
-                        target = {type: 'group', id: bind.target.groupID};
-                    }
-
-                    data.bindings.push({cluster: bind.cluster.name, target});
-                }
-
-                endpoints[endpoint.ID] = data;
-            }
-
-            return {
-                ieee_address: device.ieeeAddr,
-                type: device.type,
-                network_address: device.networkAddress,
-                supported: !!definition,
-                friendly_name: resolved.name,
-                definition: definitionPayload,
-                power_source: device.powerSource,
-                software_build_id: device.softwareBuildID,
-                date_code: device.dateCode,
-                interviewing: device.interviewing,
-                interview_completed: device.interviewCompleted,
-                endpoints,
-            };
-        });
-
-        await this.mqtt.publish('bridge/devices', stringify(devices), {retain: true, qos: 0});
-    }
-
-    async publishGroups() {
-        const groups = this.zigbee.getGroups().map((group) => {
-            const resolved = this.zigbee.resolveEntity(group);
-            return {
-                id: group.groupID,
-                friendly_name: resolved.name,
-                members: group.members.map((m) => {
-                    return {
-                        ieee_address: m.deviceIeeeAddress,
-                        endpoint: m.ID,
-                    };
-                }),
-            };
-        });
-
-        await this.mqtt.publish('bridge/groups', stringify(groups), {retain: true, qos: 0});
-    }
-}
-
-module.exports = Bridge;

lib/extension/bridge.ts

@@ -0,0 +1,697 @@
+/* eslint-disable camelcase */
+import logger from '../util/logger';
+import utils from '../util/utils';
+import * as settings from '../util/settings';
+import Transport from 'winston-transport';
+import bind from 'bind-decorator';
+import stringify from 'json-stable-stringify-without-jsonify';
+import objectAssignDeep from 'object-assign-deep';
+import {detailedDiff} from 'deep-object-diff';
+import Extension from './extension';
+import Device from '../model/device';
+import Group from '../model/group';
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/(.*)`);
+
+type Scene = {id: number, name: string};
+type DefinitionPayload = {
+    model: string, vendor: string, description: string, exposes: zhc.DefinitionExpose[], supports_ota:
+    boolean, icon: string, options: zhc.DefinitionExpose[],
+};
+
+export default class Bridge extends Extension {
+    private zigbee2mqttVersion: {commitHash: string, version: string};
+    private coordinatorVersion: zh.CoordinatorVersion;
+    private restartRequired = false;
+    private lastJoinedDeviceIeeeAddr: string;
+    private requestLookup: {[key: string]: (message: KeyValue | string) => Promise<MQTTResponse>};
+
+    override async start(): Promise<void> {
+        this.requestLookup = {
+            'device/options': this.deviceOptions,
+            'device/configure_reporting': this.deviceConfigureReporting,
+            'device/remove': this.deviceRemove,
+            'device/rename': this.deviceRename,
+            'group/add': this.groupAdd,
+            'group/options': this.groupOptions,
+            'group/remove': this.groupRemove,
+            'group/rename': this.groupRename,
+            'permit_join': this.permitJoin,
+            'restart': this.restart,
+            'touchlink/factory_reset': this.touchlinkFactoryReset,
+            'touchlink/identify': this.touchlinkIdentify,
+            'touchlink/scan': this.touchlinkScan,
+            'health_check': this.healthCheck,
+            'options': this.bridgeOptions,
+            // Below are deprecated
+            'config/last_seen': this.configLastSeen,
+            'config/homeassistant': this.configHomeAssistant,
+            'config/elapsed': this.configElapsed,
+            'config/log_level': this.configLogLevel,
+        };
+
+        const mqtt = this.mqtt;
+        class EventTransport extends Transport {
+            log(info: {message: string, level: string}, callback: () => void): void {
+                const payload = stringify({message: info.message, level: info.level});
+                mqtt.publish(`bridge/logging`, payload, {}, settings.get().mqtt.base_topic, true);
+                callback();
+            }
+        }
+
+        logger.addTransport(new EventTransport());
+
+        this.zigbee2mqttVersion = await utils.getZigbee2MQTTVersion();
+        this.coordinatorVersion = await this.zigbee.getCoordinatorVersion();
+
+        this.eventBus.onDeviceRenamed(this, () => this.publishInfo());
+        this.eventBus.onGroupMembersChanged(this, () => this.publishGroups());
+        this.eventBus.onDevicesChanged(this, () => this.publishDevices() && this.publishInfo());
+        this.eventBus.onPermitJoinChanged(this, () => !this.zigbee.isStopping() && this.publishInfo());
+        this.eventBus.onScenesChanged(this, () => {
+            this.publishDevices();
+            this.publishGroups();
+        });
+
+        // Zigbee events
+        const publishEvent = (type: string, data: KeyValue): Promise<void> =>
+            this.mqtt.publish('bridge/event', stringify({type, data}), {retain: false, qos: 0});
+        this.eventBus.onDeviceJoined(this, (data) => {
+            this.lastJoinedDeviceIeeeAddr = data.device.ieeeAddr;
+            this.publishDevices();
+            publishEvent('device_joined', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+        this.eventBus.onDeviceLeave(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_leave', {ieee_address: data.ieeeAddr, friendly_name: data.name});
+        });
+        this.eventBus.onDeviceNetworkAddressChanged(this, () => this.publishDevices());
+        this.eventBus.onDeviceInterview(this, (data) => {
+            this.publishDevices();
+            const payload: KeyValue =
+                {friendly_name: data.device.name, status: data.status, ieee_address: data.device.ieeeAddr};
+            if (data.status === 'successful') {
+                payload.supported = !!data.device.definition;
+                payload.definition = this.getDefinitionPayload(data.device);
+            }
+            publishEvent('device_interview', payload);
+        });
+        this.eventBus.onDeviceAnnounce(this, (data) => {
+            this.publishDevices();
+            publishEvent('device_announce', {friendly_name: data.device.name, ieee_address: data.device.ieeeAddr});
+        });
+
+        await this.publishInfo();
+        await this.publishDevices();
+        await this.publishGroups();
+
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+    }
+
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
+        const key = match?.[1]?.toLowerCase();
+        if (key in this.requestLookup) {
+            const message = utils.parseJSON(data.message, data.message);
+
+            try {
+                const response = await this.requestLookup[key](message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
+                logger.debug(error.stack);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    /**
+     * Requests
+     */
+
+    @bind async deviceOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        return this.changeEntityOptions('device', message);
+    }
+
+    @bind async groupOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        return this.changeEntityOptions('group', message);
+    }
+
+    @bind async bridgeOptions(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || typeof message.options !== 'object') {
+            throw new Error(`Invalid payload`);
+        }
+
+        const diff: KeyValue = detailedDiff(settings.get(), message.options);
+
+        // Remove any settings that are in the deleted.diff but not in the passed options
+        const cleanupDeleted = (options: KeyValue, deleted: KeyValue): void => {
+            for (const key of Object.keys(deleted)) {
+                if (!(key in options)) {
+                    delete deleted[key];
+                } else if (!Array.isArray(options[key])) {
+                    cleanupDeleted(options[key], deleted[key]);
+                }
+            }
+        };
+        cleanupDeleted(message.options, diff.deleted);
+
+        const newSettings = objectAssignDeep({}, diff.added, diff.updated, diff.deleted);
+
+        // deep-object-diff converts arrays to objects, set original array back here
+        const convertBackArray = (before: KeyValue, after: KeyValue): void => {
+            for (const [key, afterValue] of Object.entries(after)) {
+                const beforeValue = before[key];
+                if (Array.isArray(beforeValue)) {
+                    after[key] = beforeValue;
+                } else if (afterValue && typeof beforeValue === 'object') {
+                    convertBackArray(beforeValue, afterValue);
+                }
+            }
+        };
+        convertBackArray(message.options, newSettings);
+
+        const restartRequired = settings.apply(newSettings);
+        if (restartRequired) this.restartRequired = true;
+
+        // Apply some settings on-the-fly.
+        if (newSettings.hasOwnProperty('permit_join')) {
+            await this.zigbee.permitJoin(newSettings.permit_join);
+        }
+
+        if (newSettings.hasOwnProperty('homeassistant')) {
+            await this.enableDisableExtension(newSettings.homeassistant, 'HomeAssistant');
+        }
+
+        if (newSettings.hasOwnProperty('advanced') && newSettings.advanced.hasOwnProperty('log_level')) {
+            logger.setLevel(newSettings.advanced.log_level);
+        }
+
+        logger.info('Succesfully changed options');
+        this.publishInfo();
+        return utils.getResponse(message, {restart_required: this.restartRequired}, null);
+    }
+
+    @bind async deviceRemove(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.removeEntity('device', message);
+    }
+
+    @bind async groupRemove(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.removeEntity('group', message);
+    }
+
+    @bind async healthCheck(message: string | KeyValue): Promise<MQTTResponse> {
+        return utils.getResponse(message, {healthy: true}, null);
+    }
+
+    @bind async groupAdd(message: string | KeyValue): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('friendly_name')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const friendlyName = typeof message === 'object' ? message.friendly_name : message;
+        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : null;
+        const group = settings.addGroup(friendlyName, ID);
+        this.zigbee.createGroup(group.ID);
+        this.publishGroups();
+        return utils.getResponse(message, {friendly_name: group.friendly_name, id: group.ID}, null);
+    }
+
+    @bind async deviceRename(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.renameEntity('device', message);
+    }
+
+    @bind async groupRename(message: string | KeyValue): Promise<MQTTResponse> {
+        return this.renameEntity('group', message);
+    }
+
+    @bind async restart(message: string | KeyValue): Promise<MQTTResponse> {
+        // Wait 500 ms before restarting so response can be send.
+        setTimeout(this.restartCallback, 500);
+        logger.info('Restarting Zigbee2MQTT');
+        return utils.getResponse(message, {}, null);
+    }
+
+    @bind async permitJoin(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message === 'object' && !message.hasOwnProperty('value')) {
+            throw new Error('Invalid payload');
+        }
+
+        let value: boolean | string;
+        let time: number;
+        let device: Device = null;
+        if (typeof message === 'object') {
+            value = message.value;
+            time = message.time;
+            if (message.device) {
+                const resolved = this.zigbee.resolveEntity(message.device);
+                if (resolved instanceof Device) {
+                    device = resolved;
+                } else {
+                    throw new Error(`Device '${message.device}' does not exist`);
+                }
+            }
+        } else {
+            value = message;
+        }
+
+        if (typeof value === 'string') {
+            value = value.toLowerCase() === 'true';
+        }
+
+        await this.zigbee.permitJoin(value, device, time);
+        const response: {value: boolean, device?: string, time?: number} = {value};
+        if (device && typeof message === 'object') response.device = message.device;
+        if (time && typeof message === 'object') response.time = message.time;
+        return utils.getResponse(message, response, null);
+    }
+
+    // Deprecated
+    @bind async configLastSeen(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
+        const value = this.getValue(message);
+        if (typeof value !== 'string' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'last_seen'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configHomeAssistant(message: string | KeyValue): Promise<MQTTResponse> {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        await this.enableDisableExtension(value, 'HomeAssistant');
+        settings.set(['homeassistant'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configElapsed(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = [true, false];
+        const value = this.getValue(message);
+        if (typeof value !== 'boolean' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        settings.set(['advanced', 'elapsed'], value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    // Deprecated
+    @bind async configLogLevel(message: KeyValue | string): Promise<MQTTResponse> {
+        const allowed = ['error', 'warn', 'info', 'debug'];
+        const value = this.getValue(message) as 'error' | 'warn' | 'info' | 'debug';
+        if (typeof value !== 'string' || !allowed.includes(value)) {
+            throw new Error(`'${value}' is not an allowed value, allowed: ${allowed}`);
+        }
+
+        logger.setLevel(value);
+        this.publishInfo();
+        return utils.getResponse(message, {value}, null);
+    }
+
+    @bind async touchlinkIdentify(message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('ieee_address') ||
+            !message.hasOwnProperty('channel')) {
+            throw new Error('Invalid payload');
+        }
+
+        logger.info(`Start Touchlink identify of '${message.ieee_address}' on channel ${message.channel}`);
+        await this.zigbee.touchlinkIdentify(message.ieee_address, message.channel);
+        return utils.getResponse(message, {ieee_address: message.ieee_address, channel: message.channel}, null);
+    }
+
+    @bind async touchlinkFactoryReset(message: KeyValue | string): Promise<MQTTResponse> {
+        let result = false;
+        const payload: {ieee_address?: string, channel?: number} = {};
+        if (typeof message === 'object' && message.hasOwnProperty('ieee_address') &&
+            message.hasOwnProperty('channel')) {
+            logger.info(`Start Touchlink factory reset of '${message.ieee_address}' on channel ${message.channel}`);
+            result = await this.zigbee.touchlinkFactoryReset(message.ieee_address, message.channel);
+            payload.ieee_address = message.ieee_address;
+            payload.channel = message.channel;
+        } else {
+            logger.info('Start Touchlink factory reset of first found device');
+            result = await this.zigbee.touchlinkFactoryResetFirst();
+        }
+
+        if (result) {
+            logger.info('Successfully factory reset device through Touchlink');
+            return utils.getResponse(message, payload, null);
+        } else {
+            logger.error('Failed to factory reset device through Touchlink');
+            throw new Error('Failed to factory reset device through Touchlink');
+        }
+    }
+
+    @bind async touchlinkScan(message: KeyValue | string): Promise<MQTTResponse> {
+        logger.info('Start Touchlink scan');
+        const result = await this.zigbee.touchlinkScan();
+        const found = result.map((r) => {
+            return {ieee_address: r.ieeeAddr, channel: r.channel};
+        });
+        logger.info('Finished Touchlink scan');
+        return utils.getResponse(message, {found}, null);
+    }
+
+    /**
+     * Utils
+     */
+
+    getValue(message: KeyValue | string): string | boolean | number {
+        if (typeof message === 'object') {
+            if (!message.hasOwnProperty('value')) {
+                throw new Error('No value given');
+            }
+
+            return message.value;
+        } else {
+            return message;
+        }
+    }
+
+    async changeEntityOptions(entityType: 'device' | 'group', message: KeyValue | string): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('options')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const cleanup = (o: KeyValue): KeyValue => {
+            delete o.friendlyName; delete o.friendly_name; delete o.ID; delete o.type; delete o.devices;
+            return o;
+        };
+
+        const ID = message.id;
+        const entity = this.getEntity(entityType, ID);
+        const oldOptions = objectAssignDeep({}, cleanup(entity.settings));
+        settings.changeEntityOptions(ID, message.options);
+        const newOptions = cleanup(entity.settings);
+        await this.publishInfo();
+
+        logger.info(`Changed config for ${entityType} ${ID}`);
+
+        return utils.getResponse(message, {from: oldOptions, to: newOptions, id: ID}, null);
+    }
+
+    @bind async deviceConfigureReporting(message: string | KeyValue): Promise<MQTTResponse> {
+        if (typeof message !== 'object' || !message.hasOwnProperty('id') || !message.hasOwnProperty('cluster') ||
+            !message.hasOwnProperty('maximum_report_interval') || !message.hasOwnProperty('minimum_report_interval') ||
+            !message.hasOwnProperty('reportable_change') || !message.hasOwnProperty('attribute')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        const parsedID = utils.parseEntityID(message.id);
+        const endpoint = (this.getEntity('device', parsedID.ID) as Device).endpoint(parsedID.endpoint);
+
+        const coordinatorEndpoint = this.zigbee.firstCoordinatorEndpoint();
+        await endpoint.bind(message.cluster, coordinatorEndpoint);
+
+        await endpoint.configureReporting(message.cluster, [{
+            attribute: message.attribute, minimumReportInterval: message.minimum_report_interval,
+            maximumReportInterval: message.maximum_report_interval, reportableChange: message.reportable_change,
+        }]);
+
+        this.publishDevices();
+
+        logger.info(`Configured reporting for '${message.id}', '${message.cluster}.${message.attribute}'`);
+
+        return utils.getResponse(message, {
+            id: message.id, cluster: message.cluster, maximum_report_interval: message.maximum_report_interval,
+            minimum_report_interval: message.minimum_report_interval, reportable_change: message.reportable_change,
+            attribute: message.attribute,
+        }, null);
+    }
+
+    async renameEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
+        const deviceAndHasLast = entityType === 'device' && typeof message === 'object' && message.last === true;
+        if (typeof message !== 'object' || (!message.hasOwnProperty('from') && !deviceAndHasLast) ||
+            !message.hasOwnProperty('to')) {
+            throw new Error(`Invalid payload`);
+        }
+
+        if (deviceAndHasLast && !this.lastJoinedDeviceIeeeAddr) {
+            throw new Error('No device has joined since start');
+        }
+
+        const from = deviceAndHasLast ? this.lastJoinedDeviceIeeeAddr : message.from;
+        const to = message.to;
+        const homeAssisantRename = message.hasOwnProperty('homeassistant_rename') ?
+            message.homeassistant_rename : false;
+        const entity = this.getEntity(entityType, from);
+        const oldFriendlyName = entity.settings.friendly_name;
+
+        settings.changeFriendlyName(from, to);
+
+        // Clear retained messages
+        this.mqtt.publish(oldFriendlyName, '', {retain: true});
+
+        if (entity instanceof Device) {
+            this.publishDevices();
+            this.eventBus.emitDeviceRenamed({device: entity, homeAssisantRename, from: oldFriendlyName, to});
+        } else {
+            this.publishGroups();
+            this.publishInfo();
+        }
+
+        // Repulish entity state
+        this.publishEntityState(entity, {});
+
+        return utils.getResponse(
+            message,
+            {from: oldFriendlyName, to, homeassistant_rename: homeAssisantRename},
+            null,
+        );
+    }
+
+    async removeEntity(entityType: 'group' | 'device', message: string | KeyValue): Promise<MQTTResponse> {
+        const ID = typeof message === 'object' ? message.id : message.trim();
+        const entity = this.getEntity(entityType, ID);
+        const friendlyName = entity.name;
+        const entityID = entity.ID;
+
+        let block = false;
+        let force = false;
+        let blockForceLog = '';
+
+        if (entityType === 'device' && typeof message === 'object') {
+            block = !!message.block;
+            force = !!message.force;
+            blockForceLog = ` (block: ${block}, force: ${force})`;
+        } else if (entityType === 'group' && typeof message === 'object') {
+            force = !!message.force;
+            blockForceLog = ` (force: ${force})`;
+        }
+
+        try {
+            logger.info(`Removing ${entityType} '${entity.name}'${blockForceLog}`);
+            const ieeeAddr = entity.isDevice() && entity.ieeeAddr;
+            const name = entity.name;
+
+            if (entity instanceof Device) {
+                if (block) {
+                    settings.blockDevice(entity.ieeeAddr);
+                }
+
+                if (force) {
+                    await entity.zh.removeFromDatabase();
+                } else {
+                    await entity.zh.removeFromNetwork();
+                }
+            } else {
+                if (force) {
+                    entity.zh.removeFromDatabase();
+                } else {
+                    await entity.zh.removeFromNetwork();
+                }
+            }
+
+            // Fire event
+            if (entity instanceof Device) {
+                this.eventBus.emitDeviceRemoved({ieeeAddr, name});
+            }
+
+            // Remove from configuration.yaml
+            if (entity instanceof Device) {
+                settings.removeDevice(entityID as string);
+            } else {
+                settings.removeGroup(entityID);
+            }
+
+            // Remove from state
+            this.state.remove(entityID);
+
+            // Clear any retained messages
+            this.mqtt.publish(friendlyName, '', {retain: true});
+
+            logger.info(`Successfully removed ${entityType} '${friendlyName}'${blockForceLog}`);
+
+            if (entity instanceof Device) {
+                this.publishGroups();
+                this.publishDevices();
+                return utils.getResponse(message, {id: ID, block, force}, null);
+            } else {
+                this.publishGroups();
+                return utils.getResponse(message, {id: ID, force: force}, null);
+            }
+        } catch (error) {
+            throw new Error(
+                `Failed to remove ${entityType} '${friendlyName}'${blockForceLog} (${error})`,
+            );
+        }
+    }
+
+    getEntity(type: 'group' | 'device', ID: string): Device | Group {
+        const entity = this.zigbee.resolveEntity(ID);
+        if (!entity || entity.constructor.name.toLowerCase() !== type) {
+            throw new Error(`${utils.capitalize(type)} '${ID}' does not exist`);
+        }
+        return entity;
+    }
+
+    async publishInfo(): Promise<void> {
+        const config = objectAssignDeep({}, settings.get());
+        delete config.advanced.network_key;
+        delete config.mqtt.password;
+        config.frontend && delete config.frontend.auth_token;
+        const payload = {
+            version: this.zigbee2mqttVersion.version,
+            commit: this.zigbee2mqttVersion.commitHash,
+            coordinator: this.coordinatorVersion,
+            network: utils.toSnakeCase(await this.zigbee.getNetworkParameters()),
+            log_level: logger.getLevel(),
+            permit_join: this.zigbee.getPermitJoin(),
+            permit_join_timeout: this.zigbee.getPermitJoinTimeout(),
+            restart_required: this.restartRequired,
+            config,
+            config_schema: settings.schema,
+        };
+
+        await this.mqtt.publish(
+            'bridge/info', stringify(payload), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    private getScenes(entity: zh.Endpoint | zh.Group): Scene[] {
+        const scenes: {[id: number]: Scene} = {};
+        const endpoints = utils.isEndpoint(entity) ? [entity] : entity.members;
+        const groupID = utils.isEndpoint(entity) ? 0 : entity.groupID;
+
+        for (const endpoint of endpoints) {
+            for (const [key, data] of Object.entries(endpoint.meta?.scenes || {})) {
+                const split = key.split('_');
+                const sceneID = parseInt(split[0], 10);
+                const sceneGroupID = parseInt(split[1], 10);
+                if (sceneGroupID === groupID) {
+                    scenes[sceneID] = {id: sceneID, name: (data as KeyValue).name || `Scene ${sceneID}`};
+                }
+            }
+        }
+
+        return Object.values(scenes);
+    }
+
+    async publishDevices(): Promise<void> {
+        interface Data {
+            bindings: {cluster: string, target: {type: string, endpoint?: number, ieee_address?: string, id?: number}}[]
+            configured_reportings: {cluster: string, attribute: string | number,
+                minimum_report_interval: number, maximum_report_interval: number, reportable_change: number}[],
+            clusters: {input: string[], output: string[]}, scenes: Scene[]
+        }
+
+        const devices = this.zigbee.devices().map((device) => {
+            const endpoints: {[s: number]: Data} = {};
+            for (const endpoint of device.zh.endpoints) {
+                const data: Data = {
+                    scenes: this.getScenes(endpoint),
+                    bindings: [],
+                    configured_reportings: [],
+                    clusters: {
+                        input: endpoint.getInputClusters().map((c) => c.name),
+                        output: endpoint.getOutputClusters().map((c) => c.name),
+                    },
+                };
+
+                for (const bind of endpoint.binds) {
+                    const target = utils.isEndpoint(bind.target) ?
+                        {type: 'endpoint', ieee_address: bind.target.getDevice().ieeeAddr, endpoint: bind.target.ID} :
+                        {type: 'group', id: bind.target.groupID};
+                    data.bindings.push({cluster: bind.cluster.name, target});
+                }
+
+                for (const configuredReporting of endpoint.configuredReportings) {
+                    data.configured_reportings.push({
+                        cluster: configuredReporting.cluster.name,
+                        attribute: configuredReporting.attribute.name || configuredReporting.attribute.ID,
+                        minimum_report_interval: configuredReporting.minimumReportInterval,
+                        maximum_report_interval: configuredReporting.maximumReportInterval,
+                        reportable_change: configuredReporting.reportableChange,
+                    });
+                }
+
+                endpoints[endpoint.ID] = data;
+            }
+
+            return {
+                ieee_address: device.ieeeAddr,
+                type: device.zh.type,
+                network_address: device.zh.networkAddress,
+                supported: !!device.definition,
+                friendly_name: device.name,
+                definition: this.getDefinitionPayload(device),
+                power_source: device.zh.powerSource,
+                software_build_id: device.zh.softwareBuildID,
+                date_code: device.zh.dateCode,
+                model_id: device.zh.modelID,
+                interviewing: device.zh.interviewing,
+                interview_completed: device.zh.interviewCompleted,
+                manufacturer: device.zh.manufacturerName,
+                endpoints,
+            };
+        });
+
+        await this.mqtt.publish('bridge/devices', stringify(devices),
+            {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    async publishGroups(): Promise<void> {
+        const groups = this.zigbee.groups().map((g) => {
+            return {
+                id: g.ID,
+                friendly_name: g.ID === 901 ? 'default_bind_group' : g.name,
+                scenes: this.getScenes(g.zh),
+                members: g.zh.members.map((e) => {
+                    return {ieee_address: e.getDevice().ieeeAddr, endpoint: e.ID};
+                }),
+            };
+        });
+        await this.mqtt.publish(
+            'bridge/groups', stringify(groups), {retain: true, qos: 0}, settings.get().mqtt.base_topic, true);
+    }
+
+    getDefinitionPayload(device: Device): DefinitionPayload {
+        if (!device.definition) return null;
+        let icon = device.settings.icon ? device.settings.icon : device.definition.icon;
+        if (icon) {
+            icon = icon.replace('${zigbeeModel}', utils.sanitizeImageParameter(device.zh.modelID));
+            icon = icon.replace('${model}', utils.sanitizeImageParameter(device.definition.model));
+        }
+
+        return {
+            model: device.definition.model,
+            vendor: device.definition.vendor,
+            description: device.definition.description,
+            exposes: device.definition.exposes,
+            supports_ota: !!device.definition.ota,
+            options: device.definition.options,
+            icon,
+        };
+    }
+}

lib/extension/configure.js

@@ -1,162 +0,0 @@
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const logger = require('../util/logger');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-/**
- * This extension calls the zigbee-herdsman-converters definition configure() method
- */
-class Configure extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        this.configuring = new Set();
-        this.onReportingDisabled = this.onReportingDisabled.bind(this);
-        this.attempts = {};
-
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
-        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
-        this.eventBus.on(`reportingDisabled`, this.onReportingDisabled);
-    }
-
-    onReportingDisabled(data) {
-        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
-        const device = data.device;
-
-        const resolvedEntity = this.zigbee.resolveEntity(device);
-        if (resolvedEntity.device.meta && resolvedEntity.device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(resolvedEntity)) {
-            this.configure(resolvedEntity);
-        }
-    }
-
-    shouldConfigure(resolvedEntity) {
-        if (!resolvedEntity || !resolvedEntity.definition || !resolvedEntity.definition.configure) {
-            return false;
-        }
-
-        if (resolvedEntity.device.meta &&
-            resolvedEntity.device.meta.hasOwnProperty('configured') &&
-            resolvedEntity.device.meta.configured === resolvedEntity.definition.meta.configureKey) {
-            return false;
-        }
-
-        if (resolvedEntity.device.interviewing === true) {
-            return false;
-        }
-
-        return true;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (settings.get().advanced.legacy_api) {
-            this.mqtt.subscribe(this.legacyTopic);
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
-    async onMQTTMessage(topic, message) {
-        if (topic === this.legacyTopic) {
-            const resolvedEntity = this.zigbee.resolveEntity(message);
-            if (!resolvedEntity || resolvedEntity.type !== 'device') {
-                logger.error(`Device '${message}' does not exist`);
-                return;
-            }
-
-            if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
-                logger.warn(`Skipping configure of '${resolvedEntity.name}', device does not require this.`);
-                return;
-            }
-
-            this.configure(resolvedEntity, true);
-        } else if (topic === this.topic) {
-            message = utils.parseJSON(message, message);
-            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
-            let error = null;
-
-            const resolvedEntity = this.zigbee.resolveEntity(ID);
-            if (!resolvedEntity || resolvedEntity.type !== 'device') {
-                error = `Device '${ID}' does not exist`;
-            } else if (!resolvedEntity.definition || !resolvedEntity.definition.configure) {
-                error = `Device '${resolvedEntity.name}' cannot be configured`;
-            } else {
-                try {
-                    await this.configure(resolvedEntity, true, true);
-                } catch (e) {
-                    error = `Failed to configure (${e.message})`;
-                }
-            }
-
-            const response = utils.getResponse(message, {id: ID}, error);
-            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
-        }
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.shouldConfigure(resolvedEntity)) {
-                await this.configure(resolvedEntity);
-            }
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        const device = data.device;
-
-        if (type === 'deviceJoined' && device.meta.hasOwnProperty('configured')) {
-            delete device.meta.configured;
-            device.save();
-        }
-
-        if (this.shouldConfigure(resolvedEntity)) {
-            this.configure(resolvedEntity);
-        }
-    }
-
-    async configure(resolvedEntity, force=false, thowError=false) {
-        const device = resolvedEntity.device;
-        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
-            return false;
-        }
-
-        this.configuring.add(device.ieeeAddr);
-
-        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
-            this.attempts[device.ieeeAddr] = 0;
-        }
-
-        logger.info(`Configuring '${resolvedEntity.name}'`);
-        try {
-            await resolvedEntity.definition.configure(device, this.coordinatorEndpoint);
-            logger.info(`Successfully configured '${resolvedEntity.name}'`);
-            device.meta.configured = resolvedEntity.definition.meta.configureKey;
-            device.save();
-        } catch (error) {
-            this.attempts[device.ieeeAddr]++;
-            const attempt = this.attempts[device.ieeeAddr];
-            const msg = `Failed to configure '${resolvedEntity.name}', attempt ${attempt} (${error.stack})`;
-            logger.error(msg);
-
-            if (thowError) {
-                throw error;
-            }
-        }
-
-        this.configuring.delete(device.ieeeAddr);
-    }
-}
-
-module.exports = Configure;

lib/extension/configure.ts

@@ -0,0 +1,133 @@
+import * as settings from '../util/settings';
+import utils from '../util/utils';
+import logger from '../util/logger';
+import stringify from 'json-stable-stringify-without-jsonify';
+import zhc from 'zigbee-herdsman-converters';
+import Extension from './extension';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+/**
+ * This extension calls the zigbee-herdsman-converters definition configure() method
+ */
+export default class Configure extends Extension {
+    private configuring = new Set();
+    private attempts: {[s: string]: number} = {};
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/device/configure`;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/configure`;
+
+    @bind private async onReportingDisabled(data: eventdata.ReportingDisabled): Promise<void> {
+        // Disabling reporting unbinds some cluster which could be bound by configure, re-setup.
+        if (data.device.zh.meta?.hasOwnProperty('configured')) {
+            delete data.device.zh.meta.configured;
+            data.device.zh.save();
+        }
+
+        await this.configure(data.device, 'reporting_disabled');
+    }
+
+    @bind private async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        if (data.topic === this.legacyTopic) {
+            const device = this.zigbee.resolveEntity(data.message);
+            if (!device || !(device instanceof Device)) {
+                logger.error(`Device '${data.message}' does not exist`);
+                return;
+            }
+
+            if (!device.definition || !device.definition.configure) {
+                logger.warn(`Skipping configure of '${device.name}', device does not require this.`);
+                return;
+            }
+
+            this.configure(device, 'mqtt_message', true);
+        } else if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
+            const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
+            let error = null;
+
+            const device = this.zigbee.resolveEntity(ID);
+            if (!device || !(device instanceof Device)) {
+                error = `Device '${ID}' does not exist`;
+            } else if (!device.definition || !device.definition.configure) {
+                error = `Device '${device.name}' cannot be configured`;
+            } else {
+                try {
+                    await this.configure(device, 'mqtt_message', true, true);
+                } catch (e) {
+                    error = `Failed to configure (${e.message})`;
+                }
+            }
+
+            const response = utils.getResponse(message, {id: ID}, error);
+            await this.mqtt.publish(`bridge/response/device/configure`, stringify(response));
+        }
+    }
+
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.devices(false)) {
+            await this.configure(device, 'started');
+        }
+
+        this.eventBus.onDeviceJoined(this, (data) => {
+            if (data.device.zh.meta.hasOwnProperty('configured')) {
+                delete data.device.zh.meta.configured;
+                data.device.zh.save();
+            }
+
+            this.configure(data.device, 'zigbee_event');
+        });
+        this.eventBus.onDeviceInterview(this, (data) => this.configure(data.device, 'zigbee_event'));
+        this.eventBus.onLastSeenChanged(this, (data) => this.configure(data.device, 'zigbee_event'));
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+        this.eventBus.onReportingDisabled(this, this.onReportingDisabled);
+    }
+
+    private async configure(device: Device, event: 'started' | 'zigbee_event' | 'reporting_disabled' | 'mqtt_message',
+        force=false, thowError=false): Promise<void> {
+        if (!force) {
+            if (!device.definition?.configure || device.zh.interviewing) {
+                return;
+            }
+
+            if (device.zh.meta?.hasOwnProperty('configured') &&
+                device.zh.meta.configured === zhc.getConfigureKey(device.definition)) {
+                return;
+            }
+
+            // Only configure end devices when it is active, otherwise it will likely fails as they are sleeping.
+            if (device.zh.type === 'EndDevice' && event !== 'zigbee_event') {
+                return;
+            }
+        }
+
+        if (this.configuring.has(device.ieeeAddr) || (this.attempts[device.ieeeAddr] >= 3 && !force)) {
+            return;
+        }
+
+        this.configuring.add(device.ieeeAddr);
+
+        if (!this.attempts.hasOwnProperty(device.ieeeAddr)) {
+            this.attempts[device.ieeeAddr] = 0;
+        }
+
+        logger.info(`Configuring '${device.name}'`);
+        try {
+            await device.definition.configure(device.zh, this.zigbee.firstCoordinatorEndpoint(), logger);
+            logger.info(`Successfully configured '${device.name}'`);
+            device.zh.meta.configured = zhc.getConfigureKey(device.definition);
+            device.zh.save();
+            this.eventBus.emitDevicesChanged();
+        } catch (error) {
+            this.attempts[device.ieeeAddr]++;
+            const attempt = this.attempts[device.ieeeAddr];
+            const msg = `Failed to configure '${device.name}', attempt ${attempt} (${error.stack})`;
+            logger.error(msg);
+
+            if (thowError) {
+                throw error;
+            }
+        }
+
+        this.configuring.delete(device.ieeeAddr);
+    }
+}

lib/extension/extension.js

@@ -1,54 +0,0 @@
-class Extension {
-    /**
-     * Besides intializing variables, the constructor should do nothing!
-     *
-     * @param {Zigbee} zigbee Zigbee controller
-     * @param {MQTT} mqtt MQTT controller
-     * @param {State} state State controller
-     * @param {Function} publishEntityState Method to publish device state to MQTT.
-     * @param {EventBus} eventBus The event bus
-     */
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        this.zigbee = zigbee;
-        this.mqtt = mqtt;
-        this.state = state;
-        this.publishEntityState = publishEntityState;
-        this.eventBus = eventBus;
-    }
-
-    /**
-     * This method is called by the controller once Zigbee has been started.
-     */
-    // onZigbeeStarted() {}
-
-    /**
-     * This method is called by the controller once connected to the MQTT server.
-     */
-    // onMQTTConnected() {}
-
-    /**
-     * Is called when a Zigbee message from a device is received.
-     * @param {string} type Type of the message
-     * @param {Object} data Data of the message
-     * @param {Object?} resolvedEntity Resolved entity returned from this.zigbee.resolveEntity()
-     * @param {Object?} settingsDevice Device settings
-     */
-    // onZigbeeEvent(type, data, resolvedEntity) {}
-
-    /**
-     * Is called when a MQTT message is received
-     * @param {string} topic Topic on which the message was received
-     * @param {Object} message The received message
-     * @return {boolean} if the message was handled
-     */
-    // onMQTTMessage(topic, message) {}
-
-    /**
-     * Is called once the extension has to stop
-     */
-    stop() {
-        this.eventBus.removeListenersExtension(this.constructor.name);
-    }
-}
-
-module.exports = Extension;

lib/extension/extension.ts

@@ -0,0 +1,53 @@
+abstract class Extension {
+    protected zigbee: Zigbee;
+    protected mqtt: MQTT;
+    protected state: State;
+    protected publishEntityState: PublishEntityState;
+    protected eventBus: EventBus;
+    protected enableDisableExtension: (enable: boolean, name: string) => Promise<void>;
+    protected restartCallback: () => void;
+    protected addExtension: (extension: Extension) => void;
+
+    /**
+     * Besides intializing variables, the constructor should do nothing!
+     *
+     * @param {Zigbee} zigbee Zigbee controller
+     * @param {MQTT} mqtt MQTT controller
+     * @param {State} state State controller
+     * @param {Function} publishEntityState Method to publish device state to MQTT.
+     * @param {EventBus} eventBus The event bus
+     * @param {enableDisableExtension} enableDisableExtension Enable/disable extension method
+     * @param {restartCallback} restartCallback Restart Zigbee2MQTT
+     * @param {addExtension} addExtension Add an extension
+     */
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        this.zigbee = zigbee;
+        this.mqtt = mqtt;
+        this.state = state;
+        this.publishEntityState = publishEntityState;
+        this.eventBus = eventBus;
+        this.enableDisableExtension = enableDisableExtension;
+        this.restartCallback = restartCallback;
+        this.addExtension = addExtension;
+    }
+
+    /**
+     * Is called once the extension has to start
+     */
+    /* istanbul ignore next */
+    async start(): Promise<void> {}
+
+    /**
+     * Is called once the extension has to stop
+     */
+    async stop(): Promise<void> {
+        this.eventBus.removeListeners(this);
+    }
+
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    public adjustMessageBeforePublish(entity: Group | Device, message: KeyValue): void {}
+}
+
+export default Extension;

lib/extension/externalConverters.js

@@ -1,18 +0,0 @@
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const settings = require('../util/settings');
-const Extension = require('./extension');
-const utils = require('../util/utils');
-
-class ExternalConverters extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-
-        for (const definition of utils.getExternalConvertersDefinitions(settings)) {
-            const toAdd = {...definition};
-            delete toAdd['homeassistant'];
-            zigbeeHerdsmanConverters.addDeviceDefinition(toAdd);
-        }
-    }
-}
-
-module.exports = ExternalConverters;

lib/extension/externalConverters.ts

@@ -0,0 +1,18 @@
+import zhc from 'zigbee-herdsman-converters';
+import * as settings from '../util/settings';
+import utils from '../util/utils';
+import Extension from './extension';
+
+export default class ExternalConverters extends Extension {
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
+
+        for (const definition of utils.getExternalConvertersDefinitions(settings.get())) {
+            const toAdd = {...definition};
+            delete toAdd['homeassistant'];
+            zhc.addDeviceDefinition(toAdd);
+        }
+    }
+}

lib/extension/externalExtension.ts

@@ -0,0 +1,109 @@
+import * as settings from '../util/settings';
+import utils from '../util/utils';
+import fs from 'fs';
+import data from './../util/data';
+import path from 'path';
+import logger from './../util/logger';
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
+import Extension from './extension';
+
+const requestRegex = new RegExp(`${settings.get().mqtt.base_topic}/bridge/request/extension/(save|remove)`);
+
+export default class ExternalExtension extends Extension {
+    private requestLookup: {[s: string]: (message: KeyValue) => MQTTResponse};
+
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+        this.requestLookup = {'save': this.saveExtension, 'remove': this.removeExtension};
+        this.loadUserDefinedExtensions();
+        await this.publishExtensions();
+    }
+
+    private getExtensionsBasePath(): string {
+        return data.joinPath('extension');
+    }
+
+    private getListOfUserDefinedExtensions(): {name: string, code: string}[] {
+        const basePath = this.getExtensionsBasePath();
+        if (fs.existsSync(basePath)) {
+            return fs.readdirSync(basePath).filter((f) => f.endsWith('.js')).map((fileName) => {
+                const extensonFilePath = path.join(basePath, fileName);
+                return {'name': fileName, 'code': fs.readFileSync(extensonFilePath, 'utf-8')};
+            });
+        } else {
+            return [];
+        }
+    }
+
+    @bind private removeExtension(message: KeyValue): MQTTResponse {
+        const {name} = message;
+        const extensions = this.getListOfUserDefinedExtensions();
+        const extensionToBeRemoved = extensions.find((e) => e.name === name);
+
+        if (extensionToBeRemoved) {
+            this.enableDisableExtension(false, extensionToBeRemoved.name);
+            const basePath = this.getExtensionsBasePath();
+            const extensionFilePath = path.join(basePath, path.basename(name));
+            fs.unlinkSync(extensionFilePath);
+            this.publishExtensions();
+            logger.info(`Extension ${name} removed`);
+            return utils.getResponse(message, {}, null);
+        } else {
+            return utils.getResponse(message, {}, `Extension ${name} doesn't exists`);
+        }
+    }
+
+    @bind private saveExtension(message: KeyValue): MQTTResponse {
+        const {name, code} = message;
+        const ModuleConstructor = utils.loadModuleFromText(code) as Extension;
+        this.loadExtension(ModuleConstructor);
+        const basePath = this.getExtensionsBasePath();
+        /* istanbul ignore else */
+        if (!fs.existsSync(basePath)) {
+            fs.mkdirSync(basePath);
+        }
+        const extensonFilePath = path.join(basePath, path.basename(name));
+        fs.writeFileSync(extensonFilePath, code);
+        this.publishExtensions();
+        logger.info(`Extension ${name} loaded`);
+        return utils.getResponse(message, {}, null);
+    }
+
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const match = data.topic.match(requestRegex);
+        if (match && this.requestLookup[match[1].toLowerCase()]) {
+            const message = utils.parseJSON(data.message, data.message) as KeyValue;
+            try {
+                const response = this.requestLookup[match[1].toLowerCase()](message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            } catch (error) {
+                logger.error(`Request '${data.topic}' failed with error: '${error.message}'`);
+                const response = utils.getResponse(message, {}, error.message);
+                await this.mqtt.publish(`bridge/response/extension/${match[1]}`, stringify(response));
+            }
+        }
+    }
+
+    @bind private loadExtension(ConstructorClass: Extension): void {
+        this.enableDisableExtension(false, ConstructorClass.constructor.name);
+        // @ts-ignore
+        this.addExtension(new ConstructorClass(
+            this.zigbee, this.mqtt, this.state, this.publishEntityState, this.eventBus, settings, logger));
+    }
+
+    private loadUserDefinedExtensions(): void {
+        const extensions = this.getListOfUserDefinedExtensions();
+        extensions
+            .map(({code}) => utils.loadModuleFromText(code))
+            .map(this.loadExtension);
+    }
+
+    private async publishExtensions(): Promise<void> {
+        const extensions = this.getListOfUserDefinedExtensions();
+        await this.mqtt.publish('bridge/extensions', stringify(extensions), {
+            retain: true,
+            qos: 0,
+        }, settings.get().mqtt.base_topic, true);
+    }
+}

lib/extension/frontend.js

@@ -1,121 +0,0 @@
-const http = require('http');
-const httpProxy = require('http-proxy');
-const serveStatic = require('serve-static');
-const finalhandler = require('finalhandler');
-const Extension = require('./extension');
-const logger = require('../util/logger');
-const frontend = require('zigbee2mqtt-frontend');
-const WebSocket = require('ws');
-const url = require('url');
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-/**
- * This extension servers the frontend
- */
-class Frontend extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.onRequest = this.onRequest.bind(this);
-        this.onUpgrade = this.onUpgrade.bind(this);
-        this.mqttBaseTopic = settings.get().mqtt.base_topic;
-        this.onMQTTPublishedMessage = this.onMQTTPublishedMessage.bind(this);
-        this.mqtt.on('publishedMessage', this.onMQTTPublishedMessage);
-        this.onWebSocketConnection = this.onWebSocketConnection.bind(this);
-        this.server = http.createServer(this.onRequest);
-        this.server.on('upgrade', this.onUpgrade);
-        this.developmentServer = settings.get().frontend.development_server;
-        this.development = !!this.developmentServer;
-        this.port = settings.get().frontend.port || 8080;
-        this.retainedMessages = new Map();
-
-        if (this.development) {
-            this.proxy = httpProxy.createProxyServer({ws: true});
-        } else {
-            this.fileServer = serveStatic(frontend.getPath());
-        }
-
-        this.wss = new WebSocket.Server({noServer: true});
-        this.wss.on('connection', this.onWebSocketConnection);
-    }
-
-    onZigbeeStarted() {
-        if (this.development) {
-            logger.info(`Running frontend in development mode (${this.developmentServer})`);
-        }
-
-        this.server.listen(this.port);
-        logger.info(`Started frontend on port ${this.port}`);
-    }
-
-    async stop() {
-        for (const client of this.wss.clients) {
-            client.close();
-        }
-
-        return new Promise((resolve) => {
-            this.server.close(resolve);
-        });
-    }
-
-    onRequest(request, response) {
-        if (this.development) {
-            this.proxy.web(request, response, {target: `http://${this.developmentServer}`});
-        } else {
-            this.fileServer(request, response, finalhandler(request, response));
-        }
-    }
-
-    onUpgrade(request, socket, head) {
-        const pathname = url.parse(request.url).pathname;
-        if (pathname === '/api') {
-            const wss = this.wss;
-            wss.handleUpgrade(request, socket, head, (ws) => wss.emit('connection', ws, request));
-        } else if (this.development && pathname === '/sockjs-node') {
-            this.proxy.ws(request, socket, head, {target: `ws://${this.developmentServer}`});
-        } else {
-            socket.destroy();
-        }
-    }
-
-    onWebSocketConnection(ws) {
-        ws.on('message', (message) => {
-            if (message) {
-                const {topic, payload} = utils.parseJSON(message, message);
-                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
-            }
-        });
-
-        for (const [key, value] of this.retainedMessages) {
-            ws.send(stringify({topic: key, payload: value}));
-        }
-
-        for (const device of this.zigbee.getClients()) {
-            if (this.state.exists(device.ieeeAddr)) {
-                const resolvedEntity = this.zigbee.resolveEntity(device);
-                ws.send(stringify({topic: resolvedEntity.name, payload: this.state.get(device.ieeeAddr)}));
-            }
-        }
-    }
-
-    onMQTTPublishedMessage(data) {
-        let {topic, payload, options} = data;
-        if (topic.startsWith(`${this.mqttBaseTopic}/`)) {
-            // Send topic without base_topic
-            topic = topic.substring(this.mqttBaseTopic.length + 1);
-            payload = utils.parseJSON(payload, payload);
-            if (options.retain) {
-                this.retainedMessages.set(topic, payload);
-            }
-
-            for (const client of this.wss.clients) {
-                if (client.readyState === WebSocket.OPEN) {
-                    client.send(stringify({topic, payload}));
-                }
-            }
-        }
-    }
-}
-
-module.exports = Frontend;

lib/extension/frontend.ts

@@ -0,0 +1,139 @@
+import http from 'http';
+import serveStatic from 'serve-static';
+import finalhandler from 'finalhandler';
+import logger from '../util/logger';
+import frontend from 'zigbee2mqtt-frontend';
+import WebSocket from 'ws';
+import net from 'net';
+import url from 'url';
+import * as settings from '../util/settings';
+import utils from '../util/utils';
+import stringify from 'json-stable-stringify-without-jsonify';
+import Extension from './extension';
+import bind from 'bind-decorator';
+
+/**
+ * This extension servers the frontend
+ */
+export default class Frontend extends Extension {
+    private mqttBaseTopic = settings.get().mqtt.base_topic;
+    private host = settings.get().frontend.host || '0.0.0.0';
+    private port = settings.get().frontend.port || 8080;
+    private authToken = settings.get().frontend.auth_token || false;
+    private retainedMessages = new Map();
+    private server: http.Server;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    private fileServer: serveStatic.RequestHandler<any>;
+    private wss: WebSocket.Server = null;
+
+    constructor(zigbee: Zigbee, mqtt: MQTT, state: State, publishEntityState: PublishEntityState,
+        eventBus: EventBus, enableDisableExtension: (enable: boolean, name: string) => Promise<void>,
+        restartCallback: () => void, addExtension: (extension: Extension) => void) {
+        super(zigbee, mqtt, state, publishEntityState, eventBus, enableDisableExtension, restartCallback, addExtension);
+        this.eventBus.onMQTTMessagePublished(this, this.onMQTTPublishMessage);
+    }
+
+    override async start(): Promise<void> {
+        this.server = http.createServer(this.onRequest);
+        this.server.on('upgrade', this.onUpgrade);
+
+        /* istanbul ignore next */ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const options = {setHeaders: (res: any, path: any): void => {
+            if (path.endsWith('index.html')) {
+                res.setHeader('Cache-Control', 'no-store');
+            }
+        }};
+
+        this.fileServer = serveStatic(frontend.getPath(), options);
+        this.wss = new WebSocket.Server({noServer: true});
+        this.wss.on('connection', this.onWebSocketConnection);
+
+        this.server.listen(this.port, this.host);
+        logger.info(`Started frontend on port ${this.host}:${this.port}`);
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const client of this.wss.clients) {
+            client.send(stringify({topic: 'bridge/state', payload: 'offline'}));
+            client.terminate();
+        }
+        this.wss.close();
+        return new Promise((cb: () => void) => this.server.close(cb));
+    }
+
+    @bind private onRequest(request: http.IncomingMessage, response: http.ServerResponse): void {
+        // @ts-ignore
+        this.fileServer(request, response, finalhandler(request, response));
+    }
+
+    private authenticate(request: http.IncomingMessage, cb: (authenticate: boolean) => void): void {
+        const {query} = url.parse(request.url, true);
+        cb(!this.authToken || this.authToken === query.token);
+    }
+
+    @bind private onUpgrade(request: http.IncomingMessage, socket: net.Socket, head: Buffer): void {
+        this.wss.handleUpgrade(request, socket, head, (ws) => {
+            this.authenticate(request, (isAuthentificated) => {
+                if (isAuthentificated) {
+                    this.wss.emit('connection', ws, request);
+                } else {
+                    ws.close(4401, 'Unauthorized');
+                }
+            });
+        });
+    }
+
+    @bind private onWebSocketConnection(ws: WebSocket): void {
+        ws.on('message', (data: Buffer, isBinary: boolean) => {
+            if (!isBinary && data) {
+                const message = data.toString();
+                const {topic, payload} = JSON.parse(message);
+                this.mqtt.onMessage(`${this.mqttBaseTopic}/${topic}`, stringify(payload));
+            }
+        });
+
+        for (const [key, value] of this.retainedMessages) {
+            ws.send(stringify({topic: key, payload: value}));
+        }
+
+        for (const device of this.zigbee.devices(false)) {
+            let payload: KeyValue = {};
+            if (this.state.exists(device)) {
+                payload = {...payload, ...this.state.get(device)};
+            }
+
+            const lastSeen = settings.get().advanced.last_seen;
+            /* istanbul ignore if */
+            if (lastSeen !== 'disable') {
+                payload.last_seen = utils.formatDate(device.zh.lastSeen, lastSeen);
+            }
+
+            if (device.zh.linkquality !== undefined) {
+                payload.linkquality = device.zh.linkquality;
+            }
+
+            ws.send(stringify({topic: device.name, payload}));
+        }
+    }
+
+    @bind private onMQTTPublishMessage(data: eventdata.MQTTMessagePublished): void {
+        if (data.topic.startsWith(`${this.mqttBaseTopic}/`)) {
+            // Send topic without base_topic
+            const topic = data.topic.substring(this.mqttBaseTopic.length + 1);
+            const payload = utils.parseJSON(data.payload, data.payload);
+            if (data.options.retain) {
+                this.retainedMessages.set(topic, payload);
+            }
+
+            if (this.wss) {
+                for (const client of this.wss.clients) {
+                    /* istanbul ignore else */
+                    if (client.readyState === WebSocket.OPEN) {
+                        client.send(stringify({topic, payload}));
+                    }
+                }
+            }
+        }
+    }
+}

lib/extension/groups.js

@@ -1,343 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const Extension = require('./extension');
-const utils = require('../util/utils');
-const postfixes = utils.getEndpointNames();
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const topicRegex =
-    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
-const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
-const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
-
-class Groups extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.onStateChange = this.onStateChange.bind(this);
-        this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/group/remove_all`);
-
-            for (let step = 1; step < 20; step++) {
-                const topic = `${settings.get().mqtt.base_topic}/bridge/group/${'+/'.repeat(step)}`;
-                this.mqtt.subscribe(`${topic}remove`);
-                this.mqtt.subscribe(`${topic}add`);
-                this.mqtt.subscribe(`${topic}remove_all`);
-            }
-        }
-
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/add`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/group/members/remove_all`);
-        }
-    }
-
-    async onZigbeeStarted() {
-        this.eventBus.on('stateChange', this.onStateChange);
-        await this.syncGroupsWithSettings();
-    }
-
-    async syncGroupsWithSettings() {
-        const settingsGroups = settings.getGroups();
-        const zigbeeGroups = this.zigbee.getGroups();
-
-        const addRemoveFromGroup = async (action, deviceName, groupName, endpoint, group) => {
-            try {
-                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
-                if (action === 'remove') {
-                    await endpoint.removeFromGroup(group);
-                } else {
-                    await endpoint.addToGroup(group);
-                }
-            } catch (error) {
-                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
-            }
-        };
-
-        for (const settingGroup of settingsGroups) {
-            const groupID = settingGroup.ID;
-            const zigbeeGroup = zigbeeGroups.find((g) => g.groupID === groupID) || this.zigbee.createGroup(groupID);
-            const settingsEntity = settingGroup.devices.map((d) => {
-                const entity = this.zigbee.resolveEntity(d);
-                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendlyName}'`);
-                return entity;
-            }).filter((e) => e != null);
-
-            // In settings but not in zigbee
-            for (const entity of settingsEntity) {
-                if (!zigbeeGroup.hasMember(entity.endpoint)) {
-                    addRemoveFromGroup('add', entity.name, settingGroup.friendlyName, entity.endpoint, zigbeeGroup);
-                }
-            }
-
-            // In zigbee but not in settings
-            for (const endpoint of zigbeeGroup.members) {
-                if (!settingsEntity.find((e) => e.endpoint === endpoint)) {
-                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
-                    addRemoveFromGroup('remove', deviceName, settingGroup.friendlyName, endpoint, zigbeeGroup);
-                }
-            }
-        }
-
-        for (const zigbeeGroup of zigbeeGroups) {
-            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.groupID)) {
-                for (const endpoint of zigbeeGroup.members) {
-                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendlyName;
-                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.groupID, endpoint, zigbeeGroup);
-                }
-            }
-        }
-    }
-
-    async onStateChange(data) {
-        const reason = 'group_optimistic';
-        if (data.reason === reason) {
-            return;
-        }
-
-        const properties = ['state', 'brightness', 'color_temp', 'color'];
-        const payload = {};
-
-        properties.forEach((prop) => {
-            if (data.to.hasOwnProperty(prop)) {
-                payload[prop] = data.to[prop];
-            }
-        });
-
-        if (Object.keys(payload).length) {
-            const resolvedEntity = this.zigbee.resolveEntity(data.ID);
-            const zigbeeGroups = this.zigbee.getGroups().filter((zigbeeGroup) => {
-                const settingsGroup = settings.getGroup(zigbeeGroup.groupID);
-                return settingsGroup && settingsGroup.optimistic;
-            });
-
-            if (resolvedEntity.type === 'device') {
-                for (const zigbeeGroup of zigbeeGroups) {
-                    if (zigbeeGroup.hasMember(resolvedEntity.endpoint)) {
-                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
-                            await this.publishEntityState(zigbeeGroup.groupID, payload, reason);
-                        }
-                    }
-                }
-            } else {
-                const groupIDsToPublish = new Set();
-                for (const member of resolvedEntity.group.members) {
-                    await this.publishEntityState(member.getDevice().ieeeAddr, payload, reason);
-                    for (const zigbeeGroup of zigbeeGroups) {
-                        if (zigbeeGroup.hasMember(member)) {
-                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
-                                groupIDsToPublish.add(zigbeeGroup.groupID);
-                            }
-                        }
-                    }
-                }
-                groupIDsToPublish.delete(resolvedEntity.group.groupID);
-                for (const groupID of groupIDsToPublish) {
-                    await this.publishEntityState(groupID, payload, reason);
-                }
-            }
-        }
-    }
-
-    areAllMembersOff(zigbeeGroup) {
-        for (const member of zigbeeGroup.members) {
-            const device = member.getDevice();
-            if (this.state.exists(device.ieeeAddr)) {
-                const state = this.state.get(device.ieeeAddr);
-                if (state && state.state === 'ON') {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-
-    parseMQTTMessage(topic, message) {
-        let type = null;
-        let resolvedEntityGroup = null;
-        let resolvedEntityDevice = null;
-        let hasEndpointName = null;
-        let error = null;
-        let groupKey = null;
-        let deviceKey = null;
-        let triggeredViaLegacyApi = false;
-
-        /* istanbul ignore else */
-        const topicRegexMatch = topic.match(topicRegex);
-        const legacyTopicRegexRemoveAllMatch = topic.match(legacyTopicRegexRemoveAll);
-        const legacyTopicRegexMatch = topic.match(legacyTopicRegex);
-
-        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
-            triggeredViaLegacyApi = true;
-            if (legacyTopicRegexMatch) {
-                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]);
-                type = legacyTopicRegexMatch[2];
-
-                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
-                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
-
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        const payload = {
-                            friendly_name: message, group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists',
-                        };
-                        this.mqtt.publish(
-                            'bridge/log',
-                            stringify({type: `device_group_${type}_failed`, message: payload}),
-                        );
-                    }
-
-                    return {};
-                }
-            } else {
-                type = 'remove_all';
-            }
-
-            resolvedEntityDevice = this.zigbee.resolveEntity(message);
-            if (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device') {
-                logger.error(`Device '${message}' does not exist`);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const payload = {
-                        friendly_name: message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
-                    };
-                    this.mqtt.publish(
-                        'bridge/log',
-                        stringify({type: `device_group_${type}_failed`, message: payload}),
-                    );
-                }
-
-                return {};
-            }
-
-            hasEndpointName = postfixes.find((p) => message.endsWith(`/${p}`));
-        } else if (topicRegexMatch) {
-            type = topicRegexMatch[1];
-            message = JSON.parse(message);
-            deviceKey = message.device;
-
-            if (type !== 'remove_all') {
-                groupKey = message.group;
-                resolvedEntityGroup = this.zigbee.resolveEntity(message.group);
-                if (!resolvedEntityGroup || resolvedEntityGroup.type !== 'group') {
-                    error = `Group '${message.group}' does not exist`;
-                }
-            }
-
-            resolvedEntityDevice = this.zigbee.resolveEntity(message.device);
-            if (!error && (!resolvedEntityDevice || !resolvedEntityDevice.type === 'device')) {
-                error = `Device '${message.device}' does not exist`;
-            }
-
-            hasEndpointName = postfixes.find((p) => message.device.endsWith(`/${p}`));
-        }
-
-        return {
-            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, groupKey, deviceKey,
-            triggeredViaLegacyApi,
-        };
-    }
-
-    async onMQTTMessage(topic, message) {
-        let {
-            resolvedEntityGroup, resolvedEntityDevice, type, hasEndpointName, error, triggeredViaLegacyApi,
-            groupKey, deviceKey,
-        } = this.parseMQTTMessage(topic, message);
-        if (!type) return;
-
-        const responseData = {device: deviceKey};
-        if (groupKey) {
-            responseData.group = groupKey;
-        }
-
-        if (!error) {
-            try {
-                const keys = [
-                    `${resolvedEntityDevice.device.ieeeAddr}/${resolvedEntityDevice.endpoint.ID}`,
-                    `${resolvedEntityDevice.name}/${resolvedEntityDevice.endpoint.ID}`,
-                ];
-
-                const definition = resolvedEntityDevice.definition;
-                const endpoints = definition && definition.endpoint ?
-                    definition.endpoint(resolvedEntityDevice.device) : null;
-                const endpointName = endpoints ?
-                    Object.entries(endpoints).find((e) => e[1] === resolvedEntityDevice.endpoint.ID)[0] : null;
-
-                if (endpointName) {
-                    keys.push(`${resolvedEntityDevice.device.ieeeAddr}/${endpointName}`);
-                    keys.push(`${resolvedEntityDevice.name}/${endpointName}`);
-                }
-
-                if (!hasEndpointName) {
-                    keys.push(resolvedEntityDevice.name);
-                    keys.push(resolvedEntityDevice.device.ieeeAddr);
-                }
-
-                if (type === 'add') {
-                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
-                    await resolvedEntityDevice.endpoint.addToGroup(resolvedEntityGroup.group);
-                    settings.addDeviceToGroup(resolvedEntityGroup.settings.ID, keys);
-
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                        this.mqtt.publish(
-                            'bridge/log',
-                            stringify({type: `device_group_add`, message: payload}),
-                        );
-                    }
-                } else if (type === 'remove') {
-                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
-                    await resolvedEntityDevice.endpoint.removeFromGroup(resolvedEntityGroup.group);
-                    settings.removeDeviceFromGroup(resolvedEntityGroup.settings.ID, keys);
-
-                    /* istanbul ignore else */
-                    if (settings.get().advanced.legacy_api) {
-                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
-                        this.mqtt.publish(
-                            'bridge/log',
-                            stringify({type: `device_group_remove`, message: payload}),
-                        );
-                    }
-                } else { // remove_all
-                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
-                    await resolvedEntityDevice.endpoint.removeFromAllGroups();
-                    for (const settingsGroup of settings.getGroups()) {
-                        settings.removeDeviceFromGroup(settingsGroup.ID, keys);
-
-                        /* istanbul ignore else */
-                        if (settings.get().advanced.legacy_api) {
-                            const payload = {friendly_name: resolvedEntityDevice.name};
-                            this.mqtt.publish(
-                                'bridge/log',
-                                stringify({type: `device_group_remove_all`, message: payload}),
-                            );
-                        }
-                    }
-                }
-            } catch (e) {
-                error = `Failed to ${type} from group (${e.message})`;
-            }
-        }
-
-        if (!triggeredViaLegacyApi) {
-            const response = utils.getResponse(message, responseData, error);
-            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
-        }
-
-        if (error) {
-            logger.error(error);
-        } else {
-            this.eventBus.emit('groupMembersChanged', {group: resolvedEntityGroup});
-        }
-    }
-}
-
-module.exports = Groups;

lib/extension/groups.ts

@@ -0,0 +1,373 @@
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import utils from '../util/utils';
+import stringify from 'json-stable-stringify-without-jsonify';
+import equals from 'fast-deep-equal/es6';
+import bind from 'bind-decorator';
+import Extension from './extension';
+import Device from '../model/device';
+import Group from '../model/group';
+
+const topicRegex =
+    new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/group/members/(remove|add|remove_all)$`);
+const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/(.+)/(remove|add|remove_all)$`);
+const legacyTopicRegexRemoveAll = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/group/remove_all$`);
+
+const stateProperties: {[s: string]: (value: string, definition: zhc.Definition) => boolean} = {
+    'state': () => true,
+    'brightness': (value, definition) =>
+        !!definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'brightness')),
+    'color_temp': (value, definition) =>
+        !!definition.exposes.find((e) => e.type === 'light' && e.features.find((f) => f.name === 'color_temp')),
+    'color': (value, definition) =>
+        !!definition.exposes.find((e) => e.type === 'light' &&
+            e.features.find((f) => f.name === 'color_xy' || f.name === 'color_hs')),
+    'color_mode': (value, definition) =>
+        !!definition.exposes.find((e) => e.type === 'light' && (
+            (e.features.find((f) => f.name === `color_${value}`)) ||
+            (value === 'color_temp' && e.features.find((f) => f.name === 'color_temp')) )),
+};
+
+interface ParsedMQTTMessage {
+    type: 'remove' | 'add' | 'remove_all', resolvedEntityGroup: Group, resolvedEntityDevice: Device,
+    error: string, groupKey: string, deviceKey: string, triggeredViaLegacyApi: boolean,
+    skipDisableReporting: boolean, resolvedEntityEndpoint: zh.Endpoint,
+}
+
+export default class Groups extends Extension {
+    private legacyApi = settings.get().advanced.legacy_api;
+    private lastOptimisticState: {[s: string]: KeyValue} = {};
+
+    override async start(): Promise<void> {
+        this.eventBus.onStateChange(this, this.onStateChange);
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+        await this.syncGroupsWithSettings();
+    }
+
+    private async syncGroupsWithSettings(): Promise<void> {
+        const settingsGroups = settings.getGroups();
+        const zigbeeGroups = this.zigbee.groups();
+
+        const addRemoveFromGroup = async (action: 'add' | 'remove', deviceName: string,
+            groupName: string | number, endpoint: zh.Endpoint, group: Group): Promise<void> => {
+            try {
+                logger.info(`${action === 'add' ? 'Adding' : 'Removing'} '${deviceName}' to group '${groupName}'`);
+                if (action === 'remove') {
+                    await endpoint.removeFromGroup(group.zh);
+                } else {
+                    await endpoint.addToGroup(group.zh);
+                }
+            } catch (error) {
+                logger.error(`Failed to ${action} '${deviceName}' from '${groupName}'`);
+                logger.debug(error.stack);
+            }
+        };
+
+        for (const settingGroup of settingsGroups) {
+            const groupID = settingGroup.ID;
+            const zigbeeGroup = zigbeeGroups.find((g) => g.ID === groupID) || this.zigbee.createGroup(groupID);
+            const settingsEndpoint = settingGroup.devices.map((d) => {
+                const parsed = utils.parseEntityID(d);
+                const entity = this.zigbee.resolveEntity(parsed.ID) as Device;
+                if (!entity) logger.error(`Cannot find '${d}' of group '${settingGroup.friendly_name}'`);
+                return {'endpoint': entity?.endpoint(parsed.endpoint), 'name': entity?.name};
+            }).filter((e) => e.endpoint != null);
+
+            // In settings but not in zigbee
+            for (const entity of settingsEndpoint) {
+                if (!zigbeeGroup.zh.hasMember(entity.endpoint)) {
+                    addRemoveFromGroup('add', entity.name, settingGroup.friendly_name, entity.endpoint, zigbeeGroup);
+                }
+            }
+
+            // In zigbee but not in settings
+            for (const endpoint of zigbeeGroup.zh.members) {
+                if (!settingsEndpoint.find((e) => e.endpoint === endpoint)) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendly_name;
+                    addRemoveFromGroup('remove', deviceName, settingGroup.friendly_name, endpoint, zigbeeGroup);
+                }
+            }
+        }
+
+        for (const zigbeeGroup of zigbeeGroups) {
+            if (!settingsGroups.find((g) => g.ID === zigbeeGroup.ID)) {
+                for (const endpoint of zigbeeGroup.zh.members) {
+                    const deviceName = settings.getDevice(endpoint.getDevice().ieeeAddr).friendly_name;
+                    addRemoveFromGroup('remove', deviceName, zigbeeGroup.ID, endpoint, zigbeeGroup);
+                }
+            }
+        }
+    }
+
+    @bind async onStateChange(data: eventdata.StateChange): Promise<void> {
+        const reason = 'groupOptimistic';
+        if (data.reason === reason) {
+            return;
+        }
+
+        const payload: KeyValue = {};
+
+        let endpointName: string = null;
+        for (let [prop, value] of Object.entries(data.update)) {
+            const endpointNameMatch = utils.endpointNames.find((n) => prop.endsWith(`_${n}`));
+            if (endpointNameMatch) {
+                prop = prop.substring(0, prop.length - endpointNameMatch.length - 1);
+                endpointName = endpointNameMatch;
+            }
+
+            if (prop in stateProperties) {
+                payload[prop] = value;
+            }
+        }
+
+        if (Object.keys(payload).length) {
+            const entity = data.entity;
+            const groups = this.zigbee.groups().filter((g) => {
+                return g.settings && (!g.settings.hasOwnProperty('optimistic') || g.settings.optimistic);
+            });
+
+            if (entity instanceof Device) {
+                for (const group of groups) {
+                    if (group.zh.hasMember(entity.endpoint(endpointName)) &&
+                        !equals(this.lastOptimisticState[group.ID], payload)) {
+                        if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(group)) {
+                            this.lastOptimisticState[group.ID] = payload;
+                            await this.publishEntityState(group, payload, reason);
+                        }
+                    }
+                }
+            } else {
+                // Invalidate the last optimistic group state when group state is changed directly.
+                delete this.lastOptimisticState[entity.ID];
+
+                const groupsToPublish: Set<Group> = new Set();
+                for (const member of entity.zh.members) {
+                    const device = this.zigbee.resolveEntity(member.getDevice()) as Device;
+                    const memberPayload: KeyValue = {};
+                    Object.keys(payload).forEach((key) => {
+                        if (stateProperties[key](payload[key], device.definition)) {
+                            memberPayload[key] = payload[key];
+                        }
+                    });
+
+                    const endpointName = device.endpointName(member);
+                    if (endpointName) {
+                        Object.keys(memberPayload).forEach((key) => {
+                            memberPayload[`${key}_${endpointName}`] = memberPayload[key];
+                            delete memberPayload[key];
+                        });
+                    }
+
+                    await this.publishEntityState(device, memberPayload, reason);
+                    for (const zigbeeGroup of groups) {
+                        if (zigbeeGroup.zh.hasMember(member)) {
+                            if (!payload || payload.state !== 'OFF' || this.areAllMembersOff(zigbeeGroup)) {
+                                groupsToPublish.add(zigbeeGroup);
+                            }
+                        }
+                    }
+                }
+                groupsToPublish.delete(entity);
+                for (const group of groupsToPublish) {
+                    await this.publishEntityState(group, payload, reason);
+                }
+            }
+        }
+    }
+
+    private areAllMembersOff(group: Group): boolean {
+        for (const member of group.zh.members) {
+            const device = this.zigbee.resolveEntity(member.getDevice());
+            if (this.state.exists(device)) {
+                const state = this.state.get(device);
+                if (state && state.state === 'ON') {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    private parseMQTTMessage(data: eventdata.MQTTMessage): ParsedMQTTMessage {
+        let type: 'remove' | 'add' | 'remove_all' = null;
+        let resolvedEntityGroup: Group = null;
+        let resolvedEntityDevice: Device = null;
+        let resolvedEntityEndpoint: zh.Endpoint = null;
+        let error: string = null;
+        let groupKey: string = null;
+        let deviceKey: string = null;
+        let triggeredViaLegacyApi = false;
+        let skipDisableReporting = false;
+
+        /* istanbul ignore else */
+        const topicRegexMatch = data.topic.match(topicRegex);
+        const legacyTopicRegexRemoveAllMatch = data.topic.match(legacyTopicRegexRemoveAll);
+        const legacyTopicRegexMatch = data.topic.match(legacyTopicRegex);
+
+        if (this.legacyApi && (legacyTopicRegexMatch || legacyTopicRegexRemoveAllMatch)) {
+            triggeredViaLegacyApi = true;
+            if (legacyTopicRegexMatch) {
+                resolvedEntityGroup = this.zigbee.resolveEntity(legacyTopicRegexMatch[1]) as Group;
+                type = legacyTopicRegexMatch[2] as 'remove' | 'remove_all' | 'add';
+
+                if (!resolvedEntityGroup || !(resolvedEntityGroup instanceof Group)) {
+                    logger.error(`Group '${legacyTopicRegexMatch[1]}' does not exist`);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: data.message,
+                            group: legacyTopicRegexMatch[1], error: 'group doesn\'t exists'};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_${type}_failed`, message: payload}),
+                        );
+                    }
+
+                    return null;
+                }
+            } else {
+                type = 'remove_all';
+            }
+
+            const parsedEntity = utils.parseEntityID(data.message);
+            resolvedEntityDevice = this.zigbee.resolveEntity(parsedEntity.ID) as Device;
+            if (!resolvedEntityDevice || !(resolvedEntityDevice instanceof Device)) {
+                logger.error(`Device '${data.message}' does not exist`);
+
+                /* istanbul ignore else */
+                if (settings.get().advanced.legacy_api) {
+                    const payload = {
+                        friendly_name: data.message, group: legacyTopicRegexMatch[1], error: 'entity doesn\'t exists',
+                    };
+                    this.mqtt.publish(
+                        'bridge/log',
+                        stringify({type: `device_group_${type}_failed`, message: payload}),
+                    );
+                }
+
+                return null;
+            }
+            resolvedEntityEndpoint = resolvedEntityDevice.endpoint(parsedEntity.endpoint);
+        } else if (topicRegexMatch) {
+            type = topicRegexMatch[1] as 'remove' | 'add' | 'remove_all';
+            const message = JSON.parse(data.message);
+            deviceKey = message.device;
+            skipDisableReporting = 'skip_disable_reporting' in message ? message.skip_disable_reporting : false;
+
+            if (type !== 'remove_all') {
+                groupKey = message.group;
+                resolvedEntityGroup = this.zigbee.resolveEntity(message.group) as Group;
+                if (!resolvedEntityGroup || !(resolvedEntityGroup instanceof Group)) {
+                    error = `Group '${message.group}' does not exist`;
+                }
+            }
+
+            const parsed = utils.parseEntityID(message.device);
+            resolvedEntityDevice = this.zigbee.resolveEntity(parsed.ID) as Device;
+            if (!error && (!resolvedEntityDevice || !(resolvedEntityDevice instanceof Device))) {
+                error = `Device '${message.device}' does not exist`;
+            }
+            if (!error) {
+                resolvedEntityEndpoint = resolvedEntityDevice.endpoint(parsed.endpoint);
+            }
+        }
+
+        return {
+            resolvedEntityGroup, resolvedEntityDevice, type, error, groupKey, deviceKey,
+            triggeredViaLegacyApi, skipDisableReporting, resolvedEntityEndpoint,
+        };
+    }
+
+    @bind private async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const parsed = this.parseMQTTMessage(data);
+        if (!parsed || !parsed.type) return;
+        let {
+            resolvedEntityGroup, resolvedEntityDevice, type, error, triggeredViaLegacyApi,
+            groupKey, deviceKey, skipDisableReporting, resolvedEntityEndpoint,
+        } = parsed;
+        const message = utils.parseJSON(data.message, data.message);
+
+        const responseData: KeyValue = {device: deviceKey};
+        if (groupKey) {
+            responseData.group = groupKey;
+        }
+
+        if (!error) {
+            try {
+                const keys = [
+                    `${resolvedEntityDevice.ieeeAddr}/${resolvedEntityEndpoint.ID}`,
+                    `${resolvedEntityDevice.name}/${resolvedEntityEndpoint.ID}`,
+                ];
+
+                const endpointNameLocal = resolvedEntityDevice.endpointName(resolvedEntityEndpoint);
+                if (endpointNameLocal) {
+                    keys.push(`${resolvedEntityDevice.ieeeAddr}/${endpointNameLocal}`);
+                    keys.push(`${resolvedEntityDevice.name}/${endpointNameLocal}`);
+                }
+
+                if (!endpointNameLocal) {
+                    keys.push(resolvedEntityDevice.name);
+                    keys.push(resolvedEntityDevice.ieeeAddr);
+                }
+
+                if (type === 'add') {
+                    logger.info(`Adding '${resolvedEntityDevice.name}' to '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityEndpoint.addToGroup(resolvedEntityGroup.zh);
+                    settings.addDeviceToGroup(resolvedEntityGroup.ID.toString(), keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_add`, message: payload}),
+                        );
+                    }
+                } else if (type === 'remove') {
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from '${resolvedEntityGroup.name}'`);
+                    await resolvedEntityEndpoint.removeFromGroup(resolvedEntityGroup.zh);
+                    settings.removeDeviceFromGroup(resolvedEntityGroup.ID.toString(), keys);
+
+                    /* istanbul ignore else */
+                    if (settings.get().advanced.legacy_api) {
+                        const payload = {friendly_name: resolvedEntityDevice.name, group: resolvedEntityGroup.name};
+                        this.mqtt.publish(
+                            'bridge/log',
+                            stringify({type: `device_group_remove`, message: payload}),
+                        );
+                    }
+                } else { // remove_all
+                    logger.info(`Removing '${resolvedEntityDevice.name}' from all groups`);
+                    await resolvedEntityEndpoint.removeFromAllGroups();
+                    for (const settingsGroup of settings.getGroups()) {
+                        settings.removeDeviceFromGroup(settingsGroup.ID.toString(), keys);
+
+                        /* istanbul ignore else */
+                        if (settings.get().advanced.legacy_api) {
+                            const payload = {friendly_name: resolvedEntityDevice.name};
+                            this.mqtt.publish(
+                                'bridge/log',
+                                stringify({type: `device_group_remove_all`, message: payload}),
+                            );
+                        }
+                    }
+                }
+            } catch (e) {
+                error = `Failed to ${type} from group (${e.message})`;
+                logger.debug(e.stack);
+            }
+        }
+
+        if (!triggeredViaLegacyApi) {
+            const response = utils.getResponse(message, responseData, error);
+            await this.mqtt.publish(`bridge/response/group/members/${type}`, stringify(response));
+        }
+
+        if (error) {
+            logger.error(error);
+        } else {
+            this.eventBus.emitGroupMembersChanged({
+                group: resolvedEntityGroup, action: type, endpoint: resolvedEntityEndpoint, skipDisableReporting});
+        }
+    }
+}

lib/extension/legacy/bridgeLegacy.ts

@@ -1,41 +1,21 @@
-const settings = require('../../util/settings');
-const logger = require('../../util/logger');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const utils = require('../../util/utils');
-const assert = require('assert');
-const Extension = require('../extension');
-const stringify = require('json-stable-stringify-without-jsonify');
+import * as settings from '../../util/settings';
+import logger from '../../util/logger';
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+import utils from '../../util/utils';
+import assert from 'assert';
+import Extension from '../extension';
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
 
 const configRegex =
     new RegExp(`${settings.get().mqtt.base_topic}/bridge/config/((?:\\w+/get)|(?:\\w+/factory_reset)|(?:\\w+))`);
 const allowedLogLevels = ['error', 'warn', 'info', 'debug'];
 
-class BridgeLegacy extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
+export default class BridgeLegacy extends Extension {
+    private lastJoinedDeviceName: string = null;
+    private supportedOptions: {[s: string]: (topic: string, message: string) => Promise<void> | void};
 
-        // Bind functions
-        this.permitJoin = this.permitJoin.bind(this);
-        this.lastSeen = this.lastSeen.bind(this);
-        this.elapsed = this.elapsed.bind(this);
-        this.reset = this.reset.bind(this);
-        this.logLevel = this.logLevel.bind(this);
-        this.devices = this.devices.bind(this);
-        this.groups = this.groups.bind(this);
-        this.rename = this.rename.bind(this);
-        this.renameLast = this.renameLast.bind(this);
-        this.remove = this.remove.bind(this);
-        this.forceRemove = this.forceRemove.bind(this);
-        this.ban = this.ban.bind(this);
-        this.deviceOptions = this.deviceOptions.bind(this);
-        this.addGroup = this.addGroup.bind(this);
-        this.removeGroup = this.removeGroup.bind(this);
-        this.whitelist = this.whitelist.bind(this);
-        this.touchlinkFactoryReset = this.touchlinkFactoryReset.bind(this);
-
-        this.lastJoinedDeviceName = null;
-
-        // Set supported options
+    override async start(): Promise<void> {
         this.supportedOptions = {
             'permit_join': this.permitJoin,
             'last_seen': this.lastSeen,
@@ -57,24 +37,32 @@ class BridgeLegacy extends Extension {
             'whitelist': this.whitelist,
             'touchlink/factory_reset': this.touchlinkFactoryReset,
         };
+
+        this.eventBus.onDeviceJoined(this, (data) => this.onZigbeeEvent_('deviceJoined', data, data.device));
+        this.eventBus.onDeviceInterview(this, (data) => this.onZigbeeEvent_('deviceInterview', data, data.device));
+        this.eventBus.onDeviceAnnounce(this, (data) => this.onZigbeeEvent_('deviceAnnounce', data, data.device));
+        this.eventBus.onDeviceLeave(this, (data) => this.onZigbeeEvent_('deviceLeave', data, null));
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+
+        await this.publish();
     }
 
-    whitelist(topic, message) {
+    @bind whitelist(topic: string, message: string): void {
         try {
-            const entity = settings.getEntity(message);
+            const entity = settings.getDevice(message);
             assert(entity, `Entity '${message}' does not exist`);
-            settings.whitelistDevice(entity.ID);
-            logger.info(`Whitelisted '${entity.friendlyName}'`);
+            settings.whitelistDevice(entity.ID.toString());
+            logger.info(`Whitelisted '${entity.friendly_name}'`);
             this.mqtt.publish(
                 'bridge/log',
-                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendlyName}}),
+                stringify({type: 'device_whitelisted', message: {friendly_name: entity.friendly_name}}),
             );
         } catch (error) {
             logger.error(`Failed to whitelist '${message}' '${error}'`);
         }
     }
 
-    deviceOptions(topic, message) {
+    @bind deviceOptions(topic: string, message: string): void {
         let json = null;
         try {
             json = JSON.parse(message);
@@ -88,18 +76,18 @@ class BridgeLegacy extends Extension {
             return;
         }
 
-        const entity = settings.getEntity(json.friendly_name);
+        const entity = settings.getDevice(json.friendly_name);
         assert(entity, `Entity '${json.friendly_name}' does not exist`);
-        settings.changeEntityOptions(entity.ID, json.options);
+        settings.changeEntityOptions(entity.ID.toString(), json.options);
         logger.info(`Changed device specific options of '${json.friendly_name}' (${stringify(json.options)})`);
     }
 
-    async permitJoin(topic, message) {
+    @bind async permitJoin(topic: string, message: string): Promise<void> {
         await this.zigbee.permitJoin(message.toLowerCase() === 'true');
         this.publish();
     }
 
-    async reset(topic, message) {
+    @bind async reset(): Promise<void> {
         try {
             await this.zigbee.reset('soft');
             logger.info('Soft resetted ZNP');
@@ -108,7 +96,7 @@ class BridgeLegacy extends Extension {
         }
     }
 
-    lastSeen(topic, message) {
+    @bind lastSeen(topic: string, message: string): void {
         const allowed = ['disable', 'ISO_8601', 'epoch', 'ISO_8601_local'];
         if (!allowed.includes(message)) {
             logger.error(`${message} is not an allowed value, possible: ${allowed}`);
@@ -119,7 +107,7 @@ class BridgeLegacy extends Extension {
         logger.info(`Set last_seen to ${message}`);
     }
 
-    elapsed(topic, message) {
+    @bind elapsed(topic: string, message: string): void {
         const allowed = ['true', 'false'];
         if (!allowed.includes(message)) {
             logger.error(`${message} is not an allowed value, possible: ${allowed}`);
@@ -130,8 +118,8 @@ class BridgeLegacy extends Extension {
         logger.info(`Set elapsed to ${message}`);
     }
 
-    logLevel(topic, message) {
-        const level = message.toLowerCase();
+    @bind logLevel(topic: string, message: string): void {
+        const level = message.toLowerCase() as 'error' | 'warn' | 'info' | 'debug';
         if (allowedLogLevels.includes(level)) {
             logger.info(`Switching log level to '${level}'`);
             logger.setLevel(level);
@@ -142,30 +130,29 @@ class BridgeLegacy extends Extension {
         this.publish();
     }
 
-    async devices(topic, message) {
+    @bind async devices(topic: string): Promise<void> {
         const coordinator = await this.zigbee.getCoordinatorVersion();
-        const devices = this.zigbee.getDevices().map((device) => {
-            const payload = {
+        const devices = this.zigbee.devices().map((device) => {
+            const payload: KeyValue = {
                 ieeeAddr: device.ieeeAddr,
-                type: device.type,
-                networkAddress: device.networkAddress,
+                type: device.zh.type,
+                networkAddress: device.zh.networkAddress,
             };
 
-            if (device.type !== 'Coordinator') {
-                const definition = zigbeeHerdsmanConverters.findByDevice(device);
-                const friendlyDevice = settings.getDevice(device.ieeeAddr);
-                payload.model = definition ? definition.model : device.modelID;
+            if (device.zh.type !== 'Coordinator') {
+                const definition = zigbeeHerdsmanConverters.findByDevice(device.zh);
+                payload.model = definition ? definition.model : device.zh.modelID;
                 payload.vendor = definition ? definition.vendor : '-';
                 payload.description = definition ? definition.description : '-';
-                payload.friendly_name = friendlyDevice ? friendlyDevice.friendly_name : device.ieeeAddr;
-                payload.manufacturerID = device.manufacturerID;
-                payload.manufacturerName = device.manufacturerName;
-                payload.powerSource = device.powerSource;
-                payload.modelID = device.modelID;
-                payload.hardwareVersion = device.hardwareVersion;
-                payload.softwareBuildID = device.softwareBuildID;
-                payload.dateCode = device.dateCode;
-                payload.lastSeen = device.lastSeen;
+                payload.friendly_name = device.name;
+                payload.manufacturerID = device.zh.manufacturerID;
+                payload.manufacturerName = device.zh.manufacturerName;
+                payload.powerSource = device.zh.powerSource;
+                payload.modelID = device.zh.modelID;
+                payload.hardwareVersion = device.zh.hardwareVersion;
+                payload.softwareBuildID = device.zh.softwareBuildID;
+                payload.dateCode = device.zh.dateCode;
+                payload.lastSeen = device.zh.lastSeen;
             } else {
                 payload.friendly_name = 'Coordinator';
                 payload.softwareBuildID = coordinator.type;
@@ -177,23 +164,23 @@ class BridgeLegacy extends Extension {
         });
 
         if (topic.split('/').pop() == 'get') {
-            this.mqtt.publish(`bridge/config/devices`, stringify(devices), {});
+            this.mqtt.publish(
+                `bridge/config/devices`, stringify(devices), {}, settings.get().mqtt.base_topic, false, false,
+            );
         } else {
             this.mqtt.publish('bridge/log', stringify({type: 'devices', message: devices}));
         }
     }
 
-    groups(topic, message) {
+    @bind groups(): void {
         const payload = settings.getGroups().map((g) => {
-            const group = {...g};
-            delete group.friendlyName;
-            return group;
+            return {...g, ID: Number(g.ID)};
         });
 
         this.mqtt.publish('bridge/log', stringify({type: 'groups', message: payload}));
     }
 
-    rename(topic, message) {
+    @bind rename(topic: string, message: string): void {
         const invalid =
             `Invalid rename message format expected {"old": "friendly_name", "new": "new_name"} got ${message}`;
 
@@ -214,7 +201,7 @@ class BridgeLegacy extends Extension {
         this._renameInternal(json.old, json.new);
     }
 
-    renameLast(topic, message) {
+    @bind renameLast(topic: string, message: string): void {
         if (!this.lastJoinedDeviceName) {
             logger.error(`Cannot rename last joined device, no device has joined during this session`);
             return;
@@ -223,15 +210,15 @@ class BridgeLegacy extends Extension {
         this._renameInternal(this.lastJoinedDeviceName, message);
     }
 
-    _renameInternal(from, to) {
+    _renameInternal(from: string, to: string): void {
         try {
             const isGroup = settings.getGroup(from) !== null;
             settings.changeFriendlyName(from, to);
             logger.info(`Successfully renamed - ${from} to ${to} `);
             const entity = this.zigbee.resolveEntity(to);
-            const eventData = isGroup ? {group: entity.group} : {device: entity.device};
-            eventData.homeAssisantRename = false;
-            this.eventBus.emit(`${isGroup ? 'group' : 'device'}Renamed`, eventData);
+            if (entity.isDevice()) {
+                this.eventBus.emitDeviceRenamed({homeAssisantRename: false, from, to, device: entity});
+            }
 
             this.mqtt.publish(
                 'bridge/log',
@@ -242,7 +229,7 @@ class BridgeLegacy extends Extension {
         }
     }
 
-    addGroup(topic, message) {
+    @bind addGroup(topic: string, message: string): void {
         let id = null;
         let name = null;
         try {
@@ -271,15 +258,15 @@ class BridgeLegacy extends Extension {
         logger.info(`Added group '${name}'`);
     }
 
-    removeGroup(topic, message) {
+    @bind removeGroup(topic: string, message: string): void {
         const name = message;
-        const entity = this.zigbee.resolveEntity(message);
-        assert(entity && entity.type === 'group', `Group '${message}' does not exist`);
+        const entity = this.zigbee.resolveEntity(message) as Group;
+        assert(entity && entity.isGroup(), `Group '${message}' does not exist`);
 
         if (topic.includes('force')) {
-            entity.group.removeFromDatabase();
+            entity.zh.removeFromDatabase();
         } else {
-            entity.group.removeFromNetwork();
+            entity.zh.removeFromNetwork();
         }
         settings.removeGroup(message);
 
@@ -287,21 +274,21 @@ class BridgeLegacy extends Extension {
         logger.info(`Removed group '${name}'`);
     }
 
-    async forceRemove(topic, message) {
+    @bind async forceRemove(topic: string, message: string): Promise<void> {
         await this.removeForceRemoveOrBan('force_remove', message);
     }
 
-    async remove(topic, message) {
+    @bind async remove(topic: string, message: string): Promise<void> {
         await this.removeForceRemoveOrBan('remove', message);
     }
 
-    async ban(topic, message) {
+    @bind async ban(topic: string, message: string): Promise<void> {
         await this.removeForceRemoveOrBan('ban', message);
     }
 
-    async removeForceRemoveOrBan(action, message) {
-        const entity = this.zigbee.resolveEntity(message.trim());
-        const lookup = {
+    @bind async removeForceRemoveOrBan(action: string, message: string): Promise<void> {
+        const entity = this.zigbee.resolveEntity(message.trim()) as Device;
+        const lookup: KeyValue = {
             ban: ['banned', 'Banning', 'ban'],
             force_remove: ['force_removed', 'Force removing', 'force remove'],
             remove: ['removed', 'Removing', 'remove'],
@@ -314,31 +301,34 @@ class BridgeLegacy extends Extension {
             return;
         }
 
-        const cleanup = () => {
+        const ieeeAddr = entity.ieeeAddr;
+        const name = entity.name;
+
+        const cleanup = (): void => {
             // Fire event
-            this.eventBus.emit('deviceRemoved', {resolvedEntity: entity});
+            this.eventBus.emitDeviceRemoved({ieeeAddr, name});
 
             // Remove from configuration.yaml
-            settings.removeDevice(entity.settings.ID);
+            settings.removeDevice(entity.ieeeAddr);
 
             // Remove from state
-            this.state.remove(entity.settings.ID);
+            this.state.remove(ieeeAddr);
 
-            logger.info(`Successfully ${lookup[action][0]} ${entity.settings.friendlyName}`);
+            logger.info(`Successfully ${lookup[action][0]} ${entity.name}`);
             this.mqtt.publish('bridge/log', stringify({type: `device_${lookup[action][0]}`, message}));
         };
 
         try {
-            logger.info(`${lookup[action][1]} '${entity.settings.friendlyName}'`);
+            logger.info(`${lookup[action][1]} '${entity.name}'`);
             if (action === 'force_remove') {
-                await entity.device.removeFromDatabase();
+                await entity.zh.removeFromDatabase();
             } else {
-                await entity.device.removeFromNetwork();
+                await entity.zh.removeFromNetwork();
             }
 
             cleanup();
         } catch (error) {
-            logger.error(`Failed to ${lookup[action][2]} ${entity.settings.friendlyName} (${error})`);
+            logger.error(`Failed to ${lookup[action][2]} ${entity.name} (${error})`);
             // eslint-disable-next-line
             logger.error(`See https://www.zigbee2mqtt.io/information/mqtt_topics_and_message_structure.html#zigbee2mqttbridgeconfigremove for more info`);
 
@@ -346,34 +336,29 @@ class BridgeLegacy extends Extension {
         }
 
         if (action === 'ban') {
-            settings.banDevice(entity.settings.ID);
+            settings.banDevice(ieeeAddr);
         }
     }
 
-    async onMQTTConnected() {
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+`);
-        this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/config/+/+`);
-        await this.publish();
-    }
-
-    async onMQTTMessage(topic, message) {
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const {topic, message} = data;
         if (!topic.match(configRegex)) {
-            return false;
+            return;
         }
 
         const option = topic.match(configRegex)[1];
 
         if (!this.supportedOptions.hasOwnProperty(option)) {
-            return false;
+            return;
         }
 
         await this.supportedOptions[option](topic, message);
 
-        return true;
+        return;
     }
 
-    async publish() {
-        const info = await utils.getZigbee2mqttVersion();
+    async publish(): Promise<void> {
+        const info = await utils.getZigbee2MQTTVersion();
         const coordinator = await this.zigbee.getCoordinatorVersion();
         const topic = `bridge/config`;
         const payload = {
@@ -382,13 +367,13 @@ class BridgeLegacy extends Extension {
             coordinator,
             network: await this.zigbee.getNetworkParameters(),
             log_level: logger.getLevel(),
-            permit_join: await this.zigbee.getPermitJoin(),
+            permit_join: this.zigbee.getPermitJoin(),
         };
 
         await this.mqtt.publish(topic, stringify(payload), {retain: true, qos: 0});
     }
 
-    onZigbeeEvent(type, data, resolvedEntity) {
+    onZigbeeEvent_(type: string, data: KeyValue, resolvedEntity: Device): void {
         if (type === 'deviceJoined' && resolvedEntity) {
             this.lastJoinedDeviceName = resolvedEntity.name;
         }
@@ -436,7 +421,7 @@ class BridgeLegacy extends Extension {
         } else {
             /* istanbul ignore else */
             if (type === 'deviceLeave') {
-                const name = resolvedEntity ? resolvedEntity.name : data.ieeeAddr;
+                const name = data.ieeeAddr;
                 const meta = {friendly_name: name};
                 this.mqtt.publish(
                     'bridge/log',
@@ -446,13 +431,13 @@ class BridgeLegacy extends Extension {
         }
     }
 
-    async touchlinkFactoryReset() {
+    @bind async touchlinkFactoryReset(): Promise<void> {
         logger.info('Starting touchlink factory reset...');
         this.mqtt.publish(
             'bridge/log',
             stringify({type: `touchlink`, message: 'reset_started', meta: {status: 'started'}}),
         );
-        const result = await this.zigbee.touchlinkFactoryReset();
+        const result = await this.zigbee.touchlinkFactoryResetFirst();
 
         if (result) {
             logger.info('Successfully factory reset device through Touchlink');
@@ -469,5 +454,3 @@ class BridgeLegacy extends Extension {
         }
     }
 }
-
-module.exports = BridgeLegacy;

lib/extension/legacy/deviceGroupMembership.js

@@ -1,57 +0,0 @@
-/* istanbul ignore file */
-const settings = require('../../util/settings');
-const logger = require('../../util/logger');
-const Extension = require('../extension');
-
-const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
-
-class DeviceGroupMembership extends Extension {
-    onMQTTConnected() {
-        for (let step = 1; step < 20; step++) {
-            const topic = `${settings.get().mqtt.base_topic}/bridge/device/${'+/'.repeat(step)}get_group_membership`;
-            this.mqtt.subscribe(topic);
-        }
-    }
-
-    async onMQTTMessage(topic, message) {
-        const match = topic.match(topicRegex);
-        if (!match) {
-            return null;
-        }
-
-        const entity = this.zigbee.resolveEntity(match[1]);
-        if (!entity || entity.type !== 'device') {
-            logger.error(`Device '${match[1]}' does not exist`);
-            return;
-        }
-
-        const response = await entity.endpoint.command(
-            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {}, true,
-        );
-
-        if (!response) {
-            logger.warn(`Couldn't get group membership of ${entity.device.ieeeAddr}`);
-            return;
-        }
-
-        let {grouplist, capacity} = response;
-
-        grouplist = grouplist.map((gid) => {
-            const g = settings.getGroup(gid);
-            return g ? g.friendlyName : gid;
-        });
-
-        const msgGroupList = `${entity.device.ieeeAddr} is in groups [${grouplist}]`;
-        let msgCapacity;
-        if (capacity === 254) {
-            msgCapacity = 'it can be a part of at least 1 more group';
-        } else {
-            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
-        }
-        logger.info(`${msgGroupList} and ${msgCapacity}`);
-
-        this.publishEntityState(entity.device.ieeeAddr, {group_list: grouplist, group_capacity: capacity});
-    }
-}
-
-module.exports = DeviceGroupMembership;

lib/extension/legacy/deviceGroupMembership.ts

@@ -0,0 +1,56 @@
+/* istanbul ignore file */
+import * as settings from '../../util/settings';
+import logger from '../../util/logger';
+import utils from '../../util/utils';
+import Extension from '../extension';
+import bind from 'bind-decorator';
+import Device from '../../model/device';
+
+const topicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/device/(.+)/get_group_membership$`);
+
+export default class DeviceGroupMembership extends Extension {
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+    }
+
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const match = data.topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const parsed = utils.parseEntityID(match[1]);
+        const device = this.zigbee.resolveEntity(parsed.ID) as Device;
+        if (!device || !(device instanceof Device)) {
+            logger.error(`Device '${match[1]}' does not exist`);
+            return;
+        }
+        const endpoint = device.endpoint(parsed.endpoint);
+        const response = await endpoint.command(
+            `genGroups`, 'getMembership', {groupcount: 0, grouplist: []}, {},
+        );
+
+        if (!response) {
+            logger.warn(`Couldn't get group membership of ${device.ieeeAddr}`);
+            return;
+        }
+
+        let {grouplist, capacity} = response;
+
+        grouplist = grouplist.map((gid: string) => {
+            const g = settings.getGroup(gid);
+            return g ? g.friendly_name : gid;
+        });
+
+        const msgGroupList = `${device.ieeeAddr} is in groups [${grouplist}]`;
+        let msgCapacity;
+        if (capacity === 254) {
+            msgCapacity = 'it can be a part of at least 1 more group';
+        } else {
+            msgCapacity = `its remaining group capacity is ${capacity === 255 ? 'unknown' : capacity}`;
+        }
+        logger.info(`${msgGroupList} and ${msgCapacity}`);
+
+        this.publishEntityState(device, {group_list: grouplist, group_capacity: capacity});
+    }
+}

lib/extension/legacy/report.ts

@@ -0,0 +1,194 @@
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+import logger from '../../util/logger';
+import * as settings from '../../util/settings';
+import Extension from '../extension';
+
+const defaultConfiguration = {
+    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
+};
+
+const ZNLDP12LM = zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'ZNLDP12LM');
+
+const devicesNotSupportingReporting = [
+    zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'CC2530.ROUTER'),
+    zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'BASICZBR3'),
+    zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'ZM-CSW032-D'),
+    zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'TS0001'),
+    zigbeeHerdsmanConverters.definitions.find((d) => d.model === 'TS0115'),
+];
+
+const reportKey = 1;
+
+const getColorCapabilities = async (endpoint: zh.Endpoint): Promise<{colorTemperature: boolean, colorXY: boolean}> => {
+    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
+        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
+    }
+
+    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') as number;
+    return {
+        colorTemperature: (value & 1<<4) > 0,
+        colorXY: (value & 1<<3) > 0,
+    };
+};
+
+const clusters: {[s: string]:
+    {attribute: string, minimumReportInterval: number, maximumReportInterval: number, reportableChange: number
+        condition?: (endpoint: zh.Endpoint) => Promise<boolean>}[]} =
+{
+    'genOnOff': [
+        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
+    ],
+    'genLevelCtrl': [
+        {attribute: 'currentLevel', ...defaultConfiguration},
+    ],
+    'lightingColorCtrl': [
+        {
+            attribute: 'colorTemperature', ...defaultConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorTemperature,
+        },
+        {
+            attribute: 'currentX', ...defaultConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorXY,
+        },
+        {
+            attribute: 'currentY', ...defaultConfiguration,
+            condition: async (endpoint): Promise<boolean> => (await getColorCapabilities(endpoint)).colorXY,
+        },
+    ],
+    'closuresWindowCovering': [
+        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
+        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
+    ],
+};
+
+export default class Report extends Extension {
+    private queue: Set<string> = new Set();
+    private failed: Set<string> = new Set();
+    private enabled = settings.get().advanced.report;
+
+    shouldIgnoreClusterForDevice(cluster: string, definition: zhc.Definition): boolean {
+        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
+            // Device announces it but doesn't support it
+            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
+            return true;
+        }
+
+        return false;
+    }
+
+    async setupReporting(device: Device): Promise<void> {
+        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
+        this.queue.add(device.ieeeAddr);
+
+        const term1 = this.enabled ? 'Setup' : 'Disable';
+        const term2 = this.enabled ? 'setup' : 'disabled';
+
+        try {
+            for (const ep of device.zh.endpoints) {
+                for (const [cluster, configuration] of Object.entries(clusters)) {
+                    if (ep.supportsInputCluster(cluster) &&
+                        !this.shouldIgnoreClusterForDevice(cluster, device.definition)) {
+                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
+
+                        const items = [];
+                        for (const entry of configuration) {
+                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
+                                const toAdd = {...entry};
+                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
+                                items.push(toAdd);
+                                delete items[items.length - 1].condition;
+                            }
+                        }
+
+                        this.enabled ?
+                            await ep.bind(cluster, this.zigbee.firstCoordinatorEndpoint()) :
+                            await ep.unbind(cluster, this.zigbee.firstCoordinatorEndpoint());
+
+                        await ep.configureReporting(cluster, items);
+                        logger.info(
+                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
+                        );
+                    }
+                }
+            }
+
+            if (this.enabled) {
+                device.zh.meta.reporting = reportKey;
+            } else {
+                delete device.zh.meta.reporting;
+                this.eventBus.emitReportingDisabled({device});
+            }
+
+            this.eventBus.emitDevicesChanged();
+        } catch (error) {
+            logger.error(
+                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
+            );
+
+            this.failed.add(device.ieeeAddr);
+        }
+
+        device.zh.save();
+        this.queue.delete(device.ieeeAddr);
+    }
+
+    shouldSetupReporting(device: Device, messageType: string): boolean {
+        if (!device || !device.zh || !device.definition) return false;
+
+        // Handle messages of type endDeviceAnnce and devIncoming.
+        // This message is typically send when a device comes online after being powered off
+        // Ikea TRADFRI tend to forget their reporting after powered off.
+        // Re-setup reporting.
+        // Only resetup reporting if configuredReportings was not populated yet,
+        // else reconfigure is done in zigbee-herdsman-converters ikea.js/bulbOnEvent
+        // configuredReportings are saved since Zigbee2MQTT 1.17.0
+        // https://github.com/Koenkk/zigbee2mqtt/issues/966
+        if (this.enabled && messageType === 'deviceAnnounce' && device.isIkeaTradfri() &&
+            device.zh.endpoints.filter((e) => e.configuredReportings.length === 0).length ===
+                device.zh.endpoints.length) {
+            return true;
+        }
+
+        // These do not support reproting.
+        // https://github.com/Koenkk/zigbee-herdsman/issues/110
+        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
+        if (device.zh.manufacturerName === 'Philips' &&
+            philipsIgnoreSw.includes(device.zh.softwareBuildID)) return false;
+
+        if (device.zh.interviewing === true) return false;
+        if (device.zh.type !== 'Router' || device.zh.powerSource === 'Battery') return false;
+        // Gledopto devices don't support reporting.
+        if (devicesNotSupportingReporting.includes(device.definition) ||
+            device.definition.vendor === 'Gledopto') return false;
+
+        if (this.enabled && device.zh.meta.hasOwnProperty('reporting') &&
+            device.zh.meta.reporting === reportKey) {
+            return false;
+        }
+
+        if (!this.enabled && !device.zh.meta.hasOwnProperty('reporting')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.devices(false)) {
+            if (this.shouldSetupReporting(device, null)) {
+                await this.setupReporting(device);
+            }
+        }
+
+        this.eventBus.onDeviceAnnounce(this, (data) => this.onZigbeeEvent_('deviceAnnounce', data.device));
+        this.eventBus.onDeviceMessage(this, (data) => this.onZigbeeEvent_('dummy', data.device));
+        this.eventBus.onDeviceJoined(this, (data) => this.onZigbeeEvent_('dummy', data.device));
+        this.eventBus.onDeviceNetworkAddressChanged(this, (data) => this.onZigbeeEvent_('dummy', data.device));
+    }
+
+    async onZigbeeEvent_(type: string, device: Device): Promise<void> {
+        if (this.shouldSetupReporting(device, type)) {
+            await this.setupReporting(device);
+        }
+    }
+}

lib/extension/legacy/softReset.ts

@@ -1,33 +1,35 @@
 /* istanbul ignore file */
 // DEPRECATED
-const settings = require('../../util/settings');
-const logger = require('../../util/logger');
-const utils = require('../../util/utils');
-const Extension = require('../extension');
+import * as settings from '../../util/settings';
+import logger from '../../util/logger';
+import utils from '../../util/utils';
+import Extension from '../extension';
 
 /**
  * This extensions soft resets the ZNP after a certain timeout.
  */
-class SoftReset extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.timer = null;
-        this.timeout = utils.secondsToMilliseconds(settings.get().advanced.soft_reset_timeout);
-    }
+export default class SoftReset extends Extension {
+    private timer: NodeJS.Timer = null;
+    private timeout = utils.seconds(settings.get().advanced.soft_reset_timeout);
 
-    onZigbeeStarted() {
-        logger.debug(`Soft reset timeout set to ${utils.millisecondsToSeconds(this.timeout)} seconds`);
+    override async start(): Promise<void> {
+        logger.debug(`Soft reset timeout set to ${this.timeout / 1000} seconds`);
         this.resetTimer();
+        this.eventBus.onDeviceMessage(this, () => this.resetTimer());
+        this.eventBus.onDeviceAnnounce(this, () => this.resetTimer());
+        this.eventBus.onDeviceNetworkAddressChanged(this, () => this.resetTimer());
+        this.eventBus.onDeviceJoined(this, () => this.resetTimer());
+        this.eventBus.onDeviceInterview(this, () => this.resetTimer());
     }
 
-    clearTimer() {
+    private clearTimer(): void {
         if (this.timer) {
             clearTimeout(this.timer);
             this.timer = null;
         }
     }
 
-    resetTimer() {
+    private resetTimer(): void {
         if (this.timeout === 0) {
             return;
         }
@@ -36,7 +38,7 @@ class SoftReset extends Extension {
         this.timer = setTimeout(() => this.handleTimeout(), this.timeout);
     }
 
-    async handleTimeout() {
+    private async handleTimeout(): Promise<void> {
         logger.warn('Soft reset timeout triggered');
 
         try {
@@ -57,10 +59,4 @@ class SoftReset extends Extension {
 
         this.resetTimer();
     }
-
-    onZigbeeEvent() {
-        this.resetTimer();
-    }
 }
-
-module.exports = SoftReset;

lib/extension/networkMap.ts

@@ -1,27 +1,36 @@
-const settings = require('../util/settings');
-const utils = require('../util/utils');
-const logger = require('../util/logger');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
+import * as settings from '../util/settings';
+import utils from '../util/utils';
+import logger from '../util/logger';
+import stringify from 'json-stable-stringify-without-jsonify';
+import Extension from './extension';
+import bind from 'bind-decorator';
+
+interface Link {
+    source: {ieeeAddr: string, networkAddress: number}, target: {ieeeAddr: string, networkAddress: number},
+    linkquality: number, depth: number, routes: zh.RoutingTableEntry[],
+    sourceIeeeAddr: string, targetIeeeAddr: string, sourceNwkAddr: number, lqi: number, relationship: number,
+}
+
+interface Topology {
+    nodes: {
+        ieeeAddr: string, friendlyName: string, type: string, networkAddress: number, manufacturerName: string,
+        modelID: string, failed: string[], lastSeen: number,
+        definition: {model: string, vendor: string, supports: string, description: string}}[],
+    links: Link[],
+}
 
 /**
  * This extension creates a network map
  */
-class NetworkMap extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
+export default class NetworkMap extends Extension {
+    private legacyApi = settings.get().advanced.legacy_api;
+    private legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
+    private legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
+    private topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
+    private supportedFormats: {[s: string]: (topology: Topology) => KeyValue | string};
 
-        this.legacyApi = settings.get().advanced.legacy_api;
-        this.legacyTopic = `${settings.get().mqtt.base_topic}/bridge/networkmap`;
-        this.legacyTopicRoutes = `${settings.get().mqtt.base_topic}/bridge/networkmap/routes`;
-        this.topic = `${settings.get().mqtt.base_topic}/bridge/request/networkmap`;
-
-        // Bind
-        this.raw = this.raw.bind(this);
-        this.graphviz = this.graphviz.bind(this);
-        this.plantuml = this.plantuml.bind(this);
-
-        // Set supported formats
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
         this.supportedFormats = {
             'raw': this.raw,
             'graphviz': this.graphviz,
@@ -29,33 +38,22 @@ class NetworkMap extends Extension {
         };
     }
 
-    onMQTTConnected() {
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
         /* istanbul ignore else */
         if (this.legacyApi) {
-            this.mqtt.subscribe(this.legacyTopic);
-            this.mqtt.subscribe(this.legacyTopicRoutes);
-        }
-
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(this.topic);
-        }
-    }
-
-    async onMQTTMessage(topic, message) {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            if ((topic === this.legacyTopic || topic === this.legacyTopicRoutes) &&
-                this.supportedFormats.hasOwnProperty(message)) {
-                const includeRoutes = topic === this.legacyTopicRoutes;
+            if ((data.topic === this.legacyTopic || data.topic === this.legacyTopicRoutes) &&
+                this.supportedFormats.hasOwnProperty(data.message)) {
+                const includeRoutes = data.topic === this.legacyTopicRoutes;
                 const topology = await this.networkScan(includeRoutes);
-                const converted = this.supportedFormats[message](topology);
-                this.mqtt.publish(`bridge/networkmap/${message}`, converted, {});
+                let converted = this.supportedFormats[data.message](topology);
+                converted = data.message === 'raw' ? stringify(converted) : converted;
+                this.mqtt.publish(`bridge/networkmap/${data.message}`, converted as string, {});
             }
         }
 
-        if (settings.get().experimental.new_api && topic === this.topic) {
+        if (data.topic === this.topic) {
+            const message = utils.parseJSON(data.message, data.message);
             try {
-                message = utils.parseJSON(message, message);
                 const type = typeof message === 'object' ? message.type : message;
                 if (!this.supportedFormats.hasOwnProperty(type)) {
                     throw new Error(`Type '${type}' not supported, allowed are: ${Object.keys(this.supportedFormats)}`);
@@ -77,11 +75,11 @@ class NetworkMap extends Extension {
         }
     }
 
-    raw(topology) {
-        return stringify(topology);
+    @bind raw(topology: Topology): KeyValue {
+        return topology;
     }
 
-    graphviz(topology) {
+    @bind graphviz(topology: Topology): string {
         const colors = settings.get().map_options.graphviz.colors;
 
         let text = 'digraph G {\nnode[shape=record];\n';
@@ -95,7 +93,7 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             labels.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? `failed: ${node.failed.join(',')}` : ''),
             );
 
@@ -113,7 +111,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'relative');
+                lastSeen = utils.formatDate(date, 'relative') as string;
             }
 
             labels.push(lastSeen);
@@ -143,7 +141,7 @@ class NetworkMap extends Extension {
                     (!e.routes.length) ? 'penwidth=0.5, ' : 'penwidth=2, ';
                 const lineWeight = (!e.routes.length) ? `weight=0, color="${colors.line.inactive}", ` :
                     `weight=1, color="${colors.line.active}", `;
-                const textRoutes = e.routes.map((r) => r.destinationAddress);
+                const textRoutes = e.routes.map((r) => utils.toNetworkAddressHex(r.destinationAddress));
                 const lineLabels = (!e.routes.length) ? `label="${e.linkquality}"` :
                     `label="${e.linkquality} (routes: ${textRoutes.join(',')})"`;
                 text += `  "${node.ieeeAddr}" -> "${e.target.ieeeAddr}"`;
@@ -156,7 +154,7 @@ class NetworkMap extends Extension {
         return text.replace(/\0/g, '');
     }
 
-    plantuml(topology) {
+    @bind plantuml(topology: Topology): string {
         const text = [];
 
         text.push(`' paste into: https://www.planttext.com/`);
@@ -171,14 +169,14 @@ class NetworkMap extends Extension {
 
             // Add the device short network address, ieeaddr and scan note (if any)
             text.push(
-                `${node.ieeeAddr} (${node.networkAddress})` +
+                `${node.ieeeAddr} (${utils.toNetworkAddressHex(node.networkAddress)})` +
                 ((node.failed && node.failed.length) ? ` failed: ${node.failed.join(',')}` : ''),
             );
 
             // Add the device model
             if (node.type !== 'Coordinator') {
                 text.push(`---`);
-                const definition = this.zigbee.resolveEntity(node.ieeeAddr).definition;
+                const definition = (this.zigbee.resolveEntity(node.ieeeAddr) as Device).definition;
                 if (definition) {
                     text.push(`${definition.vendor} ${definition.description} (${definition.model})`);
                 } else {
@@ -191,7 +189,7 @@ class NetworkMap extends Extension {
             let lastSeen = 'unknown';
             const date = node.type === 'Coordinator' ? Date.now() : node.lastSeen;
             if (date) {
-                lastSeen = utils.formatDate(date, 'relative');
+                lastSeen = utils.formatDate(date, 'relative') as string;
             }
             text.push(`---`);
             text.push(lastSeen);
@@ -214,54 +212,71 @@ class NetworkMap extends Extension {
         return text.join(`\n`);
     }
 
-    async networkScan(includeRoutes) {
+    async networkScan(includeRoutes: boolean): Promise<Topology> {
         logger.info(`Starting network scan (includeRoutes '${includeRoutes}')`);
-        const devices = this.zigbee.getDevices().filter((d) => d.type !== 'GreenPower');
-        const lqis = new Map();
-        const routingTables = new Map();
-        const failed = new Map();
+        const devices = this.zigbee.devices().filter((d) => d.zh.type !== 'GreenPower');
+        const lqis: Map<Device, zh.LQI> = new Map();
+        const routingTables: Map<Device, zh.RoutingTable> = new Map();
+        const failed: Map<Device, string[]> = new Map();
 
-        for (const device of devices.filter((d) => d.type != 'EndDevice')) {
+        for (const device of devices.filter((d) => d.zh.type != 'EndDevice')) {
             failed.set(device, []);
-            const resolvedEntity = this.zigbee.resolveEntity(device);
+            await utils.sleep(1); // sleep 1 second between each scan to reduce stress on network.
+
+            const doRequest = async <T>(request: () => Promise<T>, firstAttempt = true): Promise<T> => {
+                try {
+                    return await request();
+                } catch (error) {
+                    if (!firstAttempt) {
+                        throw error;
+                    } else {
+                        // Network is possibly congested, sleep 5 seconds to let the network settle.
+                        await utils.sleep(5);
+                        return doRequest(request, false);
+                    }
+                }
+            };
+
             try {
-                const result = await device.lqi();
+                const result = await doRequest<zh.LQI>(async () => device.zh.lqi());
                 lqis.set(device, result);
-                logger.debug(`LQI succeeded for '${resolvedEntity.name}'`);
+                logger.debug(`LQI succeeded for '${device.name}'`);
             } catch (error) {
                 failed.get(device).push('lqi');
-                logger.error(`Failed to execute LQI for '${resolvedEntity.name}'`);
+                logger.error(`Failed to execute LQI for '${device.name}'`);
             }
 
             if (includeRoutes) {
                 try {
-                    const result = await device.routingTable();
+                    const result = await doRequest(async () => device.zh.routingTable());
                     routingTables.set(device, result);
-                    logger.debug(`Routing table succeeded for '${resolvedEntity.name}'`);
+                    logger.debug(`Routing table succeeded for '${device.name}'`);
                 } catch (error) {
                     failed.get(device).push('routingTable');
-                    logger.error(`Failed to execute routing table for '${resolvedEntity.name}'`);
+                    logger.error(`Failed to execute routing table for '${device.name}'`);
                 }
             }
         }
 
         logger.info(`Network scan finished`);
 
-        const networkMap = {nodes: [], links: []};
+        const topology: Topology = {nodes: [], links: []};
         // Add nodes
         for (const device of devices) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            const definition = resolvedEntity.definition ? {
-                model: resolvedEntity.definition.model,
-                vendor: resolvedEntity.definition.vendor,
-                description: resolvedEntity.definition.description,
-                supports: resolvedEntity.definition.supports,
+            const definition = device.definition ? {
+                model: device.definition.model,
+                vendor: device.definition.vendor,
+                description: device.definition.description,
+                supports: Array.from(new Set((device.definition.exposes).map((e) => {
+                    return e.hasOwnProperty('name') ? e.name :
+                        `${e.type} (${e.features.map((f) => f.name).join(', ')})`;
+                }))).join(', '),
             } : null;
 
-            networkMap.nodes.push({
-                ieeeAddr: device.ieeeAddr, friendlyName: resolvedEntity.name, type: device.type,
-                networkAddress: device.networkAddress, manufacturerName: device.manufacturerName,
-                modelID: device.modelID, failed: failed.get(device), lastSeen: device.lastSeen,
+            topology.nodes.push({
+                ieeeAddr: device.ieeeAddr, friendlyName: device.name, type: device.zh.type,
+                networkAddress: device.zh.networkAddress, manufacturerName: device.zh.manufacturerName,
+                modelID: device.zh.modelID, failed: failed.get(device), lastSeen: device.zh.lastSeen,
                 definition,
             });
         }
@@ -274,9 +289,16 @@ class NetworkMap extends Extension {
                     continue;
                 }
 
-                const link = {
+                // Some Xiaomi devices return 0x00 as the neighbor ieeeAddr (obviously not correct).
+                // Determine the correct ieeeAddr based on the networkAddress.
+                const neighborDevice = this.zigbee.deviceByNetworkAddress(neighbor.networkAddress);
+                if (neighbor.ieeeAddr === '0x0000000000000000' && neighborDevice) {
+                    neighbor.ieeeAddr = neighborDevice.ieeeAddr;
+                }
+
+                const link: Link = {
                     source: {ieeeAddr: neighbor.ieeeAddr, networkAddress: neighbor.networkAddress},
-                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.networkAddress},
+                    target: {ieeeAddr: device.ieeeAddr, networkAddress: device.zh.networkAddress},
                     linkquality: neighbor.linkquality, depth: neighbor.depth, routes: [],
                     // DEPRECATED:
                     sourceIeeeAddr: neighbor.ieeeAddr, targetIeeeAddr: device.ieeeAddr,
@@ -290,12 +312,10 @@ class NetworkMap extends Extension {
                         .filter((t) => t.status === 'ACTIVE' && t.nextHop === neighbor.networkAddress);
                 }
 
-                networkMap.links.push(link);
+                topology.links.push(link);
             }
         });
 
-        return networkMap;
+        return topology;
     }
 }
-
-module.exports = NetworkMap;

lib/extension/onEvent.js

@@ -1,38 +0,0 @@
-const Extension = require('./extension');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-
-/**
- * This extension calls the zigbee-herdsman-converters onEvent.
- */
-class OnEvent extends Extension {
-    async onZigbeeStarted() {
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            this.callOnEvent(resolvedEntity, 'start', {});
-        }
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        if (resolvedEntity && resolvedEntity.type === 'device') {
-            this.callOnEvent(resolvedEntity, type, data);
-        }
-    }
-
-    async stop() {
-        super.stop();
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            this.callOnEvent(resolvedEntity, 'stop', {});
-        }
-    }
-
-    callOnEvent(resolvedEntity, type, data) {
-        zigbeeHerdsmanConverters.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
-
-        if (resolvedEntity.definition && resolvedEntity.definition.onEvent) {
-            resolvedEntity.definition.onEvent(type, data, resolvedEntity.device, resolvedEntity.settings);
-        }
-    }
-}
-
-module.exports = OnEvent;

lib/extension/onEvent.ts

@@ -0,0 +1,42 @@
+import zhc from 'zigbee-herdsman-converters';
+import Extension from './extension';
+
+/**
+ * This extension calls the zigbee-herdsman-converters onEvent.
+ */
+export default class OnEvent extends Extension {
+    override async start(): Promise<void> {
+        for (const device of this.zigbee.devices(false)) {
+            await this.callOnEvent(device, 'start', {});
+        }
+
+        this.eventBus.onDeviceMessage(this, (data) => this.callOnEvent(data.device, 'message', this.convertData(data)));
+        this.eventBus.onDeviceJoined(this,
+            (data) => this.callOnEvent(data.device, 'deviceJoined', this.convertData(data)));
+        this.eventBus.onDeviceInterview(this,
+            (data) => this.callOnEvent(data.device, 'deviceInterview', this.convertData(data)));
+        this.eventBus.onDeviceAnnounce(this,
+            (data) => this.callOnEvent(data.device, 'deviceAnnounce', this.convertData(data)));
+        this.eventBus.onDeviceNetworkAddressChanged(this,
+            (data) => this.callOnEvent(data.device, 'deviceNetworkAddressChanged', this.convertData(data)));
+    }
+
+    private convertData(data: KeyValue): KeyValue {
+        return {...data, device: data.device.zh};
+    }
+
+    override async stop(): Promise<void> {
+        super.stop();
+        for (const device of this.zigbee.devices(false)) {
+            await this.callOnEvent(device, 'stop', {});
+        }
+    }
+
+    private async callOnEvent(device: Device, type: string, data: KeyValue): Promise<void> {
+        zhc.onEvent(type, data, device.zh);
+
+        if (device.definition?.onEvent) {
+            await device.definition.onEvent(type, data, device.zh, device.settings);
+        }
+    }
+}

lib/extension/otaUpdate.ts

@@ -1,71 +1,73 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const stringify = require('json-stable-stringify-without-jsonify');
-const utils = require('../util/utils');
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import stringify from 'json-stable-stringify-without-jsonify';
+import utils from '../util/utils';
+import tradfriOTA from 'zigbee-herdsman-converters/lib/ota/tradfri';
+import Extension from './extension';
+import bind from 'bind-decorator';
+import Device from '../model/device';
+
+type UpdateState = 'updating' | 'idle' | 'available';
+interface UpdatePayload {
+    // eslint-disable-next-line camelcase
+    update: {progress?: number, remaining?: number, state: UpdateState}, update_available?: boolean}
+
 const legacyTopicRegex = new RegExp(`^${settings.get().mqtt.base_topic}/bridge/ota_update/.+$`);
 const topicRegex =
     new RegExp(`^${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/(update|check)`, 'i');
 
-const Extension = require('./extension');
-const MINUTES_10 = 1000 * 60 * 10;
+export default class OTAUpdate extends Extension {
+    private inProgress = new Set();
+    private lastChecked: {[s: string]: number} = {};
+    private legacyApi = settings.get().advanced.legacy_api;
 
-class OTAUpdate extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.inProgress = new Set();
-        this.lastChecked = {};
-        this.legacyApi = settings.get().advanced.legacy_api;
-    }
-
-    onMQTTConnected() {
-        /* istanbul ignore else */
-        if (this.legacyApi) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/ota_update/update`);
+    override async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+        this.eventBus.onDeviceMessage(this, this.onZigbeeEvent);
+        if (settings.get().advanced.ikea_ota_use_test_url) {
+            tradfriOTA.useTestURL();
         }
 
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/check`);
-            this.mqtt.subscribe(`${settings.get().mqtt.base_topic}/bridge/request/device/ota_update/update`);
-        }
-
-        for (const device of this.zigbee.getClients()) {
+        for (const device of this.zigbee.devices(false)) {
             // In case Zigbee2MQTT is restared during an update, progress and remaining values are still in state.
             // remove them.
             this.removeProgressAndRemainingFromState(device);
         }
     }
 
-    removeProgressAndRemainingFromState(device) {
-        this.state.removeKey(device.ieeeAddr, ['update', 'progress']);
-        this.state.removeKey(device.ieeeAddr, ['update', 'remaining']);
+    private removeProgressAndRemainingFromState(device: Device): void {
+        delete this.state.get(device)?.update?.progress;
+        delete this.state.get(device)?.update?.remaining;
     }
 
-    async onZigbeeEvent(type, data, resolvedEntity) {
-        if (data.type !== 'commandQueryNextImageRequest' || !resolvedEntity || !resolvedEntity.definition) return;
+    @bind private async onZigbeeEvent(data: eventdata.DeviceMessage): Promise<void> {
+        if (settings.get().ota.disable_automatic_update_check) return;
+        if (data.type !== 'commandQueryNextImageRequest' || !data.device.definition) return;
+        logger.debug(`Device '${data.device.name}' requested OTA`);
 
-        const supportsOTA = resolvedEntity.definition.hasOwnProperty('ota');
+        const supportsOTA = data.device.definition.hasOwnProperty('ota');
         if (supportsOTA) {
             // When a device does a next image request, it will usually do it a few times after each other
-            // with only 10 - 60 seconds inbetween. It doesn' make sense to check for a new update
-            // each time.
+            // with only 10 - 60 seconds inbetween. It doesn't make sense to check for a new update
+            // each time, so this interval can be set by the user. The default is 1,440 minutes (one day).
+            const updateCheckInterval = settings.get().ota.update_check_interval * 1000 * 60;
             const check = this.lastChecked.hasOwnProperty(data.device.ieeeAddr) ?
-                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > MINUTES_10 : true;
+                (Date.now() - this.lastChecked[data.device.ieeeAddr]) > updateCheckInterval : true;
             if (!check || this.inProgress.has(data.device.ieeeAddr)) return;
 
             this.lastChecked[data.device.ieeeAddr] = Date.now();
-            const available = await resolvedEntity.definition.ota.isUpdateAvailable(data.device, logger, data.data);
+            const available = await data.device.definition.ota.isUpdateAvailable(
+                data.device.zh, logger, data.data);
             const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
-            this.publishEntityState(data.device.ieeeAddr, payload);
+            this.publishEntityState(data.device, payload);
 
             if (available) {
-                const message = `Update available for '${resolvedEntity.settings.friendly_name}'`;
+                const message = `Update available for '${data.device.name}'`;
                 logger.info(message);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: 'available', device: resolvedEntity.settings.friendly_name};
+                    const meta = {status: 'available', device: data.device.name};
                     this.mqtt.publish(
                         'bridge/log',
                         stringify({type: `ota_update`, message, meta}),
@@ -77,22 +79,23 @@ class OTAUpdate extends Extension {
         // Respond to the OTA request:
         // - In case we don't support OTA: respond with NO_IMAGE_AVAILABLE (0x98) (so the client stops requesting OTAs)
         // - In case we do support OTA: respond with ABORT (0x95) as we don't want to update now.
-        const endpoint = data.device.endpoints.find((e) => e.supportsOutputCluster('genOta'));
+        const endpoint = data.device.zh.endpoints.find((e) => e.supportsOutputCluster('genOta'));
         if (endpoint) {
             // Some devices send OTA requests without defining OTA cluster as input cluster.
             await endpoint.commandResponse('genOta', 'queryNextImageResponse', {status: supportsOTA ? 0x95 : 0x98});
         }
     }
 
-    async readSoftwareBuildIDAndDateCode(device, update) {
+    private async readSoftwareBuildIDAndDateCode(device: Device, update: boolean):
+        Promise<{softwareBuildID: string, dateCode: string}> {
         try {
-            const endpoint = device.endpoints.find((e) => e.supportsInputCluster('genBasic'));
+            const endpoint = device.zh.endpoints.find((e) => e.supportsInputCluster('genBasic'));
             const result = await endpoint.read('genBasic', ['dateCode', 'swBuildId']);
 
             if (update) {
-                device.softwareBuildID = result.swBuildId;
-                device.dateCode = result.dateCode;
-                device.save();
+                device.zh.softwareBuildID = result.swBuildId;
+                device.zh.dateCode = result.dateCode;
+                device.zh.save();
             }
 
             return {softwareBuildID: result.swBuildId, dateCode: result.dateCode};
@@ -101,61 +104,57 @@ class OTAUpdate extends Extension {
         }
     }
 
-    getEntityPublishPayload(state, progress=null, remaining=null) {
-        const payload = {};
+    private getEntityPublishPayload(state: UpdateState, progress: number=null, remaining: number=null): UpdatePayload {
+        const payload: UpdatePayload = {update: {state}};
+        if (progress !== null) payload.update.progress = progress;
+        if (remaining !== null) payload.update.remaining = Math.round(remaining);
 
         /* istanbul ignore else */
         if (this.legacyApi) {
             payload.update_available = state === 'available';
         }
 
-        /* istanbul ignore else */
-        if (settings.get().experimental.new_api) {
-            payload.update = {state};
-            if (progress !== null) payload.update.progress = progress;
-            if (remaining !== null) payload.update.remaining = Math.round(remaining);
-        }
-
         return payload;
     }
 
-    async onMQTTMessage(topic, message) {
-        if ((!this.legacyApi || !topic.match(legacyTopicRegex)) && !topic.match(topicRegex)) {
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        if ((!this.legacyApi || !data.topic.match(legacyTopicRegex)) && !data.topic.match(topicRegex)) {
             return null;
         }
 
-        message = utils.parseJSON(message, message);
-        const ID = typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message;
-        const resolvedEntity = this.zigbee.resolveEntity(ID);
-        const type = topic.substring(topic.lastIndexOf('/') + 1);
-        const responseData = {id: ID};
+        const message = utils.parseJSON(data.message, data.message);
+        const ID = (typeof message === 'object' && message.hasOwnProperty('id') ? message.id : message) as string;
+        const device = this.zigbee.resolveEntity(ID);
+        const type = data.topic.substring(data.topic.lastIndexOf('/') + 1);
+        const responseData: {id: string, updateAvailable?: boolean, from?: string, to?: string}= {id: ID};
         let error = null;
+        let errorStack = null;
 
-        if (!resolvedEntity || resolvedEntity.type !== 'device') {
+        if (!(device instanceof Device)) {
             error = `Device '${ID}' does not exist`;
-        } else if (!resolvedEntity.definition || !resolvedEntity.definition.ota) {
-            error = `Device '${resolvedEntity.name}' does not support OTA updates`;
+        } else if (!device.definition || !device.definition.ota) {
+            error = `Device '${device.name}' does not support OTA updates`;
 
             /* istanbul ignore else */
             if (settings.get().advanced.legacy_api) {
-                const meta = {status: `not_supported`, device: resolvedEntity.name};
+                const meta = {status: `not_supported`, device: device.name};
                 this.mqtt.publish(
                     'bridge/log',
                     stringify({type: `ota_update`, message: error, meta}),
                 );
             }
-        } else if (this.inProgress.has(resolvedEntity.device.ieeeAddr)) {
-            error = `Update or check for update already in progress for '${resolvedEntity.name}'`;
+        } else if (this.inProgress.has(device.ieeeAddr)) {
+            error = `Update or check for update already in progress for '${device.name}'`;
         } else {
-            this.inProgress.add(resolvedEntity.device.ieeeAddr);
+            this.inProgress.add(device.ieeeAddr);
 
             if (type === 'check') {
-                const msg = `Checking if update available for '${resolvedEntity.name}'`;
+                const msg = `Checking if update available for '${device.name}'`;
                 logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `checking_if_available`, device: resolvedEntity.name};
+                    const meta = {status: `checking_if_available`, device: device.name};
                     this.mqtt.publish(
                         'bridge/log',
                         stringify({type: `ota_update`, message: msg, meta}),
@@ -163,15 +162,13 @@ class OTAUpdate extends Extension {
                 }
 
                 try {
-                    const available = await resolvedEntity.definition.ota.isUpdateAvailable(
-                        resolvedEntity.device, logger,
-                    );
-                    const msg = `${available ? 'Update' : 'No update'} available for '${resolvedEntity.name}'`;
+                    const available = await device.definition.ota.isUpdateAvailable(device.zh, logger);
+                    const msg = `${available ? 'Update' : 'No update'} available for '${device.name}'`;
                     logger.info(msg);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: available ? 'available' : 'not_available', device: resolvedEntity.name};
+                        const meta = {status: available ? 'available' : 'not_available', device: device.name};
                         this.mqtt.publish(
                             'bridge/log',
                             stringify({type: `ota_update`, message: msg, meta}),
@@ -179,15 +176,16 @@ class OTAUpdate extends Extension {
                     }
 
                     const payload = this.getEntityPublishPayload(available ? 'available' : 'idle');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
-                    this.lastChecked[resolvedEntity.device.ieeeAddr] = Date.now();
+                    this.publishEntityState(device, payload);
+                    this.lastChecked[device.ieeeAddr] = Date.now();
                     responseData.updateAvailable = available;
                 } catch (e) {
-                    error = `Failed to check if update available for '${resolvedEntity.name}' (${e.message})`;
+                    error = `Failed to check if update available for '${device.name}' (${e.message})`;
+                    errorStack = e.stack;
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `check_failed`, device: resolvedEntity.name};
+                        const meta = {status: `check_failed`, device: device.name};
                         this.mqtt.publish(
                             'bridge/log',
                             stringify({type: `ota_update`, message: error, meta}),
@@ -195,75 +193,77 @@ class OTAUpdate extends Extension {
                     }
                 }
             } else { // type === 'update'
-                const msg = `Updating '${resolvedEntity.name}' to latest firmware`;
+                const msg = `Updating '${device.name}' to latest firmware`;
                 logger.info(msg);
 
                 /* istanbul ignore else */
                 if (settings.get().advanced.legacy_api) {
-                    const meta = {status: `update_in_progress`, device: resolvedEntity.name};
+                    const meta = {status: `update_in_progress`, device: device.name};
                     this.mqtt.publish(
                         'bridge/log',
-                        stringify({type: `ota_update`, msg, meta}),
+                        stringify({type: `ota_update`, message: msg, meta}),
                     );
                 }
 
                 try {
-                    const onProgress = (progress, remaining) => {
-                        let msg = `Update of '${resolvedEntity.name}' at ${progress.toFixed(2)}%`;
+                    const onProgress = (progress: number, remaining: number): void => {
+                        let msg = `Update of '${device.name}' at ${progress.toFixed(2)}%`;
                         if (remaining) {
-                            msg += `, +- ${Math.round(remaining / 60)} minutes remaining`;
+                            msg += `, ≈ ${Math.round(remaining / 60)} minutes remaining`;
                         }
 
                         logger.info(msg);
 
                         const payload = this.getEntityPublishPayload('updating', progress, remaining);
-                        this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                        this.publishEntityState(device, payload);
 
                         /* istanbul ignore else */
                         if (settings.get().advanced.legacy_api) {
-                            const meta = {status: `update_progress`, device: resolvedEntity.name, progress};
+                            const meta = {status: `update_progress`, device: device.name, progress};
                             this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: msg, meta}));
                         }
                     };
 
-                    const from_ = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, false);
-                    await resolvedEntity.definition.ota.updateToLatest(resolvedEntity.device, logger, onProgress);
-                    const to = await this.readSoftwareBuildIDAndDateCode(resolvedEntity.device, true);
+                    const from_ = await this.readSoftwareBuildIDAndDateCode(device, false);
+                    await device.definition.ota.updateToLatest(device.zh, logger, onProgress);
+                    const to = await this.readSoftwareBuildIDAndDateCode(device, true);
                     const [fromS, toS] = [stringify(from_), stringify(to)];
-                    const msg = `Finished update of '${resolvedEntity.name}'` +
+                    const msg = `Finished update of '${device.name}'` +
                         (to ? `, from '${fromS}' to '${toS}'` : ``);
                     logger.info(msg);
-                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    this.removeProgressAndRemainingFromState(device);
                     const payload = this.getEntityPublishPayload('idle');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.publishEntityState(device, payload);
                     responseData.from = from_ ? utils.toSnakeCase(from_) : null;
                     responseData.to = to ? utils.toSnakeCase(to) : null;
-                    this.eventBus.emit(`devicesChanged`);
+                    this.eventBus.emitDevicesChanged();
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_succeeded`, device: resolvedEntity.name, from: from_, to};
+                        const meta = {status: `update_succeeded`, device: device.name, from: from_, to};
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message, meta}));
                     }
                 } catch (e) {
-                    error = `Update of '${resolvedEntity.name}' failed (${e.message})`;
+                    logger.debug(`Update of '${device.name}' failed (${e})`);
+                    error = `Update of '${device.name}' failed (${e.message})`;
+                    errorStack = e.stack;
 
-                    this.removeProgressAndRemainingFromState(resolvedEntity.device);
+                    this.removeProgressAndRemainingFromState(device);
                     const payload = this.getEntityPublishPayload('available');
-                    this.publishEntityState(resolvedEntity.device.ieeeAddr, payload);
+                    this.publishEntityState(device, payload);
 
                     /* istanbul ignore else */
                     if (settings.get().advanced.legacy_api) {
-                        const meta = {status: `update_failed`, device: resolvedEntity.name};
+                        const meta = {status: `update_failed`, device: device.name};
                         this.mqtt.publish('bridge/log', stringify({type: `ota_update`, message: error, meta}));
                     }
                 }
             }
 
-            this.inProgress.delete(resolvedEntity.device.ieeeAddr);
+            this.inProgress.delete(device.ieeeAddr);
         }
 
-        const triggeredViaLegacyApi = topic.match(legacyTopicRegex);
+        const triggeredViaLegacyApi = data.topic.match(legacyTopicRegex);
         if (!triggeredViaLegacyApi) {
             const response = utils.getResponse(message, responseData, error);
             await this.mqtt.publish(`bridge/response/device/ota_update/${type}`, stringify(response));
@@ -271,8 +271,7 @@ class OTAUpdate extends Extension {
 
         if (error) {
             logger.error(error);
+            errorStack && logger.debug(errorStack);
         }
     }
 }
-
-module.exports = OTAUpdate;

lib/extension/publish.js

@@ -1,290 +0,0 @@
-
-const settings = require('../util/settings');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const assert = require('assert');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const topicRegex = new RegExp(`^(.+?)(?:/(${utils.getEndpointNames().join('|')}))?/(get|set)(?:/(.+))?`);
-const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
-
-const groupConverters = [
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_alert,
-    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
-    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
-    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
-    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
-    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
-];
-
-class EntityPublish extends Extension {
-    onMQTTConnected() {
-        // Subscribe to topics.
-        const baseTopic = settings.get().mqtt.base_topic;
-        for (let step = 1; step < 20; step++) {
-            const topic = `${baseTopic}/${'+/'.repeat(step)}`;
-            this.mqtt.subscribe(`${topic}set`);
-            this.mqtt.subscribe(`${topic}set/+`);
-            this.mqtt.subscribe(`${topic}get`);
-            this.mqtt.subscribe(`${topic}get/+`);
-        }
-    }
-
-    parseTopic(topic) {
-        const match = topic.match(topicRegex);
-        if (!match) {
-            return null;
-        }
-
-        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
-        // If we didn't replace base_topic we received something we don't care about
-        if (ID === match[1] || ID.match(/bridge/)) {
-            return null;
-        }
-
-        return {ID: ID, endpointName: match[2] || '', type: match[3], attribute: match[4]};
-    }
-
-    async onMQTTMessage(topic, message) {
-        topic = this.parseTopic(topic);
-        if (!topic) {
-            return false;
-        }
-
-        const entityKey = `${topic.ID}` + (topic.endpointName ? `/${topic.endpointName}` : '');
-        const resolvedEntity = this.zigbee.resolveEntity(entityKey);
-
-        if (!resolvedEntity) {
-            /* istanbul ignore else */
-            if (settings.get().advanced.legacy_api) {
-                const message = {friendly_name: entityKey};
-                this.mqtt.publish(
-                    'bridge/log',
-                    stringify({type: `entity_not_found`, message}),
-                );
-            }
-
-            logger.error(`Entity '${entityKey}' is unknown`);
-            return;
-        }
-
-        // Get entity details
-        let converters = null;
-        let target = null;
-        let options = {};
-        let device = null;
-        let definition = null;
-        let membersState = null;
-
-        assert(resolvedEntity.type === 'device' || resolvedEntity.type === 'group');
-        if (resolvedEntity.type === 'device') {
-            if (!resolvedEntity.definition) {
-                logger.warn(`Device with modelID '${resolvedEntity.device.modelID}' is not supported.`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
-                return;
-            }
-
-            device = resolvedEntity.device;
-            definition = resolvedEntity.definition;
-            target = resolvedEntity.endpoint;
-            converters = resolvedEntity.definition.toZigbee;
-            options = resolvedEntity.settings;
-        } else {
-            target = resolvedEntity.group;
-            options = resolvedEntity.settings;
-            definition = resolvedEntity.group.members
-                .map((e) => zigbeeHerdsmanConverters.findByDevice(e.getDevice())).filter((d) => d);
-            converters = new Set(groupConverters);
-            for (const d of definition) {
-                d.toZigbee.forEach(converters.add, converters);
-            }
-            converters = [...converters];
-            membersState = {};
-            for (const member of resolvedEntity.group.members) {
-                const ieeeAddr = member.getDevice().ieeeAddr;
-                membersState[ieeeAddr] = this.state.get(ieeeAddr);
-            }
-        }
-
-        // Convert the MQTT message to a Zigbee message.
-        let json = {};
-        if (topic.hasOwnProperty('attribute') && topic.attribute) {
-            try {
-                json[topic.attribute] = JSON.parse(message);
-            } catch (e) {
-                json[topic.attribute] = message;
-            }
-        } else {
-            try {
-                json = JSON.parse(message);
-            } catch (e) {
-                if (stateValues.includes(message.toLowerCase())) {
-                    json = {state: message};
-                } else {
-                    logger.error(`Invalid JSON '${message}', skipping...`);
-                    return false;
-                }
-            }
-        }
-        if (!json) {
-            logger.error(`Invalid JSON '${message}', skipping...`);
-            return;
-        }
-
-        /**
-         * Home Assistant always publishes 'state', even when e.g. only setting
-         * the color temperature. This would lead to 2 zigbee publishes, where the first one
-         * (state) is probably unecessary.
-         */
-        const entityState = this.state.get(resolvedEntity.settings.ID) || {};
-        if (settings.get().homeassistant) {
-            const hasColorTemp = json.hasOwnProperty('color_temp');
-            const hasColor = json.hasOwnProperty('color');
-            const hasBrightness = json.hasOwnProperty('brightness');
-            const isOn = entityState && entityState.state === 'ON' ? true : false;
-            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
-                delete json.state;
-                logger.debug('Skipping state because of Home Assistant');
-            }
-        }
-
-        /**
-         * Order state & brightness based on current bulb state
-         *
-         * Not all bulbs support setting the color/color_temp while it is off
-         * this results in inconsistant behavior between different vendors.
-         *
-         * bulb on => move state & brightness to the back
-         * bulb off => move state & brightness to the front
-         */
-        const entries = Object.entries(json);
-        const sorter = typeof json.state === 'string' && json.state.toLowerCase() === 'off' ? 1 : -1;
-        entries.sort((a, b) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
-
-        // For each attribute call the corresponding converter
-        const usedConverters = {};
-        for (let [key, value] of entries) {
-            let endpointName = topic.endpointName;
-            let actualTarget = target;
-
-            // When the key has a endpointName included (e.g. state_right), this will override the target.
-            if (resolvedEntity.type === 'device' && key.includes('_')) {
-                const underscoreIndex = key.lastIndexOf('_');
-                const possibleEndpointName = key.substring(underscoreIndex + 1, key.length);
-                if (utils.getEndpointNames().includes(possibleEndpointName)) {
-                    endpointName = possibleEndpointName;
-                    key = key.substring(0, underscoreIndex);
-                    const device = target.getDevice();
-                    actualTarget = device.getEndpoint(definition.endpoint(device)[endpointName]);
-
-                    if (!actualTarget) {
-                        logger.error(`Device '${resolvedEntity.name}' has no endpoint '${endpointName}'`);
-                        continue;
-                    }
-                }
-            }
-
-            const endpointOrGroupID = actualTarget.constructor.name == 'Group' ? actualTarget.groupID : actualTarget.ID;
-            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
-            const converter = converters.find((c) => c.key.includes(key));
-
-            if (usedConverters[endpointOrGroupID].includes(converter)) {
-                // Use a converter only once (e.g. light_onoff_brightness converters can convert state and brightness)
-                continue;
-            }
-
-            if (!converter) {
-                logger.error(`No converter available for '${key}' (${json[key]})`);
-                continue;
-            }
-
-            // Converter didn't return a result, skip
-            const meta = {
-                endpoint_name: endpointName,
-                options,
-                message: json,
-                logger,
-                device,
-                state: entityState,
-                membersState,
-                mapped: definition,
-            };
-
-            try {
-                if (topic.type === 'set' && converter.convertSet) {
-                    logger.debug(`Publishing '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
-                    const result = await converter.convertSet(actualTarget, key, value, meta);
-                    if (result && result.state) {
-                        const msg = result.state;
-
-                        if (endpointName) {
-                            for (const key of ['state', 'brightness', 'color', 'color_temp']) {
-                                if (msg.hasOwnProperty(key)) {
-                                    msg[`${key}_${endpointName}`] = msg[key];
-                                    delete msg[key];
-                                }
-                            }
-                        }
-
-                        this.publishEntityState(resolvedEntity.settings.ID, msg);
-                    }
-
-                    if (result && result.membersState) {
-                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
-                            this.publishEntityState(ieeeAddr, state);
-                        }
-                    }
-
-                    // It's possible for devices to get out of sync when writing an attribute that's not reportable.
-                    // So here we re-read the value after a specified timeout, this timeout could for example be the
-                    // transition time of a color change or for forcing a state read for devices that don't
-                    // automatically report a new state when set.
-                    // When reporting is requested for a device (report: true in device-specific settings) we won't
-                    // ever issue a read here, as we assume the device will properly report changes.
-                    // Only do this when the retrieve_state option is enabled for this device.
-                    if (
-                        resolvedEntity.type === 'device' && result && result.hasOwnProperty('readAfterWriteTime') &&
-                        resolvedEntity.settings.retrieve_state
-                    ) {
-                        setTimeout(() => converter.convertGet(actualTarget, key, meta), result.readAfterWriteTime);
-                    }
-                } else if (topic.type === 'get' && converter.convertGet) {
-                    logger.debug(`Publishing get '${topic.type}' '${key}' to '${resolvedEntity.name}'`);
-                    await converter.convertGet(actualTarget, key, meta);
-                } else {
-                    logger.error(`No converter available for '${topic.type}' '${key}' (${json[key]})`);
-                    continue;
-                }
-            } catch (error) {
-                const message =
-                    `Publish '${topic.type}' '${key}' to '${resolvedEntity.name}' failed: '${error}'`;
-                logger.error(message);
-                logger.debug(error.stack);
-
-                /* istanbul ignore else */
-                if (settings.get().advanced.legacy_api) {
-                    const meta = {friendly_name: resolvedEntity.name};
-                    this.mqtt.publish(
-                        'bridge/log',
-                        stringify({type: `zigbee_publish_error`, message, meta}),
-                    );
-                }
-            }
-
-            usedConverters[endpointOrGroupID].push(converter);
-        }
-
-        return true;
-    }
-}
-
-module.exports = EntityPublish;

lib/extension/publish.ts

@@ -0,0 +1,301 @@
+
+import * as settings from '../util/settings';
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+import logger from '../util/logger';
+import utils from '../util/utils';
+import Extension from './extension';
+import stringify from 'json-stable-stringify-without-jsonify';
+import Group from '../model/group';
+import Device from '../model/device';
+import bind from 'bind-decorator';
+
+const topicRegex = new RegExp(`^(.+?)(?:/(${utils.endpointNames.join('|')}|\\d+))?/(get|set)(?:/(.+))?`);
+const propertyEndpointRegex = new RegExp(`^(.*)_(${utils.endpointNames.join('|')}|\\d+)$`);
+const stateValues = ['on', 'off', 'toggle', 'open', 'close', 'stop', 'lock', 'unlock'];
+const sceneConverterKeys = ['scene_store', 'scene_add', 'scene_remove', 'scene_remove_all'];
+
+// Legacy: don't provide default converters anymore, this is required by older z2m installs not saving group members
+const defaultGroupConverters = [
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_onoff_brightness,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_color_colortemp,
+    zigbeeHerdsmanConverters.toZigbeeConverters.effect,
+    zigbeeHerdsmanConverters.toZigbeeConverters.ignore_transition,
+    zigbeeHerdsmanConverters.toZigbeeConverters.cover_position_tilt,
+    zigbeeHerdsmanConverters.toZigbeeConverters.thermostat_occupied_heating_setpoint,
+    zigbeeHerdsmanConverters.toZigbeeConverters.tint_scene,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_brightness_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_step,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_colortemp_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_move,
+    zigbeeHerdsmanConverters.toZigbeeConverters.light_hue_saturation_step,
+];
+
+interface ParsedTopic {ID: string, endpoint: string, attribute: string, type: 'get' | 'set'}
+
+export default class Publish extends Extension {
+    async start(): Promise<void> {
+        this.eventBus.onMQTTMessage(this, this.onMQTTMessage);
+    }
+
+    parseTopic(topic: string): ParsedTopic | null {
+        const match = topic.match(topicRegex);
+        if (!match) {
+            return null;
+        }
+
+        const ID = match[1].replace(`${settings.get().mqtt.base_topic}/`, '');
+        // If we didn't replace base_topic we received something we don't care about
+        if (ID === match[1] || ID.match(/bridge/)) {
+            return null;
+        }
+
+        return {ID: ID, endpoint: match[2], type: match[3] as 'get' | 'set', attribute: match[4]};
+    }
+
+    parseMessage(parsedTopic: ParsedTopic, data: eventdata.MQTTMessage): KeyValue | null {
+        if (parsedTopic.attribute) {
+            try {
+                return {[parsedTopic.attribute]: JSON.parse(data.message)};
+            } catch (e) {
+                return {[parsedTopic.attribute]: data.message};
+            }
+        } else {
+            try {
+                return JSON.parse(data.message);
+            } catch (e) {
+                if (stateValues.includes(data.message.toLowerCase())) {
+                    return {state: data.message};
+                } else {
+                    return null;
+                }
+            }
+        }
+    }
+
+    legacyLog(payload: KeyValue): void {
+        /* istanbul ignore else */
+        if (settings.get().advanced.legacy_api) {
+            this.mqtt.publish('bridge/log', stringify(payload));
+        }
+    }
+
+    legacyRetrieveState(re: Device | Group, converter: zhc.ToZigbeeConverter, result: zhc.ToZigbeeConverterResult,
+        target: zh.Endpoint | zh.Group, key: string, meta: zhc.ToZigbeeConverterGetMeta): void {
+        // It's possible for devices to get out of sync when writing an attribute that's not reportable.
+        // So here we re-read the value after a specified timeout, this timeout could for example be the
+        // transition time of a color change or for forcing a state read for devices that don't
+        // automatically report a new state when set.
+        // When reporting is requested for a device (report: true in device-specific settings) we won't
+        // ever issue a read here, as we assume the device will properly report changes.
+        // Only do this when the retrieve_state option is enabled for this device.
+        // retrieve_state == decprecated
+        if (re instanceof Device && result && result.hasOwnProperty('readAfterWriteTime') &&
+            re.settings.retrieve_state
+        ) {
+            setTimeout(() => converter.convertGet(target, key, meta), result.readAfterWriteTime);
+        }
+    }
+
+    updateMessageHomeAssistant(message: KeyValue, entityState: KeyValue): void {
+        /**
+         * Home Assistant always publishes 'state', even when e.g. only setting
+         * the color temperature. This would lead to 2 zigbee publishes, where the first one
+         * (state) is probably unecessary.
+         */
+        if (settings.get().homeassistant) {
+            const hasColorTemp = message.hasOwnProperty('color_temp');
+            const hasColor = message.hasOwnProperty('color');
+            const hasBrightness = message.hasOwnProperty('brightness');
+            const isOn = entityState.state === 'ON' ? true : false;
+            if (isOn && (hasColorTemp || hasColor) && !hasBrightness) {
+                delete message.state;
+                logger.debug('Skipping state because of Home Assistant');
+            }
+        }
+    }
+
+    @bind async onMQTTMessage(data: eventdata.MQTTMessage): Promise<void> {
+        const parsedTopic = this.parseTopic(data.topic);
+        if (!parsedTopic) return;
+
+        const re = this.zigbee.resolveEntity(parsedTopic.ID);
+        if (re == null) {
+            this.legacyLog({type: `entity_not_found`, message: {friendly_name: parsedTopic.ID}});
+            logger.error(`Entity '${parsedTopic.ID}' is unknown`);
+            return;
+        }
+
+        // Get entity details
+        const definition = re instanceof Device ? re.definition : re.membersDefinitions();
+        if (definition == null) {
+            logger.error(`Cannot publish to unsupported device '${re.name}'`);
+            return;
+        }
+        const target = re instanceof Group ? re.zh : re.endpoint(parsedTopic.endpoint);
+        if (target == null) {
+            logger.error(`Device '${re.name}' has no endpoint '${parsedTopic.endpoint}'`);
+            return;
+        }
+        const device = re instanceof Device ? re.zh : null;
+        const entitySettings = re.settings;
+        const entityState = this.state.get(re) || {};
+        const membersState = re instanceof Group ?
+            Object.fromEntries(re.zh.members.map((e) => [e.getDevice().ieeeAddr,
+                this.state.get(this.zigbee.resolveEntity(e.getDevice().ieeeAddr))])) : null;
+        let converters: zhc.ToZigbeeConverter[];
+        {
+            if (Array.isArray(definition)) {
+                const c = new Set(definition.map((d) => d.toZigbee).flat());
+                if (c.size == 0) converters = defaultGroupConverters;
+                else converters = Array.from(c);
+            } else {
+                converters = definition.toZigbee;
+            }
+        }
+
+        // Convert the MQTT message to a Zigbee message.
+        const message = this.parseMessage(parsedTopic, data);
+        if (message == null) {
+            logger.error(`Invalid message '${message}', skipping...`);
+            return;
+        }
+        this.updateMessageHomeAssistant(message, entityState);
+
+        /**
+         * Order state & brightness based on current bulb state
+         *
+         * Not all bulbs support setting the color/color_temp while it is off
+         * this results in inconsistant behavior between different vendors.
+         *
+         * bulb on => move state & brightness to the back
+         * bulb off => move state & brightness to the front
+         */
+        const entries = Object.entries(message);
+        const sorter = typeof message.state === 'string' && message.state.toLowerCase() === 'off' ? 1 : -1;
+        entries.sort((a) => (['state', 'brightness', 'brightness_percent'].includes(a[0]) ? sorter : sorter * -1));
+
+        // For each attribute call the corresponding converter
+        const usedConverters: {[s: number]: zhc.ToZigbeeConverter[]} = {};
+        const toPublish: {[s: number | string]: KeyValue} = {};
+        const toPublishEntity: {[s: number | string]: Device | Group} = {};
+        const addToToPublish = (entity: Device | Group, payload: KeyValue): void => {
+            const ID = entity.ID;
+            if (!(ID in toPublish)) {
+                toPublish[ID] = {};
+                toPublishEntity[ID] = entity;
+            }
+            toPublish[ID] = {...toPublish[ID], ...payload};
+        };
+
+        for (let [key, value] of entries) {
+            let endpointName = parsedTopic.endpoint;
+            let localTarget = target;
+            let endpointOrGroupID = utils.isEndpoint(target) ? target.ID : target.groupID;
+
+            // When the key has a endpointName included (e.g. state_right), this will override the target.
+            const propertyEndpointMatch = key.match(propertyEndpointRegex);
+            if (re instanceof Device && propertyEndpointMatch) {
+                endpointName = propertyEndpointMatch[2];
+                key = propertyEndpointMatch[1];
+                localTarget = re.endpoint(endpointName);
+                if (localTarget == null) {
+                    logger.error(`Device '${re.name}' has no endpoint '${endpointName}'`);
+                    continue;
+                }
+                endpointOrGroupID = localTarget.ID;
+            }
+
+            if (!usedConverters.hasOwnProperty(endpointOrGroupID)) usedConverters[endpointOrGroupID] = [];
+            const converter = converters.find((c) => c.key.includes(key));
+
+            if (parsedTopic.type === 'set' && usedConverters[endpointOrGroupID].includes(converter)) {
+                // Use a converter for set only once
+                // (e.g. light_onoff_brightness converters can convert state and brightness)
+                continue;
+            }
+
+            if (!converter) {
+                logger.error(`No converter available for '${key}' (${message[key]})`);
+                continue;
+            }
+
+            // If the endpoint_name name is a nubmer, try to map it to a friendlyName
+            if (!isNaN(Number(endpointName)) && re.isDevice() && utils.isEndpoint(localTarget) &&
+                re.endpointName(localTarget)) {
+                endpointName = re.endpointName(localTarget);
+            }
+
+            // Converter didn't return a result, skip
+            const meta = {endpoint_name: endpointName, options: entitySettings, message: {...message}, logger, device,
+                state: entityState, membersState, mapped: definition};
+
+            // Strip endpoint name from meta.message properties.
+            if (endpointName) {
+                for (const [key, value] of Object.entries(meta.message)) {
+                    if (key.endsWith(endpointName)) {
+                        delete meta.message[key];
+                        const keyWithoutEndpoint = key.substring(0, key.length - endpointName.length - 1);
+                        meta.message[keyWithoutEndpoint] = value;
+                    }
+                }
+            }
+
+            try {
+                if (parsedTopic.type === 'set' && converter.convertSet) {
+                    logger.debug(`Publishing '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    const result = await converter.convertSet(localTarget, key, value, meta);
+                    const optimistic = !entitySettings.hasOwnProperty('optimistic') || entitySettings.optimistic;
+                    if (result && result.state && optimistic) {
+                        const msg = result.state;
+
+                        if (endpointName) {
+                            for (const key of Object.keys(msg)) {
+                                msg[`${key}_${endpointName}`] = msg[key];
+                                delete msg[key];
+                            }
+                        }
+
+                        // filter out attribute listed in filtered_optimistic
+                        entitySettings.filtered_optimistic?.forEach((a) => delete msg[a]);
+                        addToToPublish(re, msg);
+                    }
+
+                    if (result && result.membersState && optimistic) {
+                        for (const [ieeeAddr, state] of Object.entries(result.membersState)) {
+                            addToToPublish(this.zigbee.resolveEntity(ieeeAddr), state);
+                        }
+                    }
+
+                    this.legacyRetrieveState(re, converter, result, localTarget, key, meta);
+                } else if (parsedTopic.type === 'get' && converter.convertGet) {
+                    logger.debug(`Publishing get '${parsedTopic.type}' '${key}' to '${re.name}'`);
+                    await converter.convertGet(localTarget, key, meta);
+                } else {
+                    logger.error(`No converter available for '${parsedTopic.type}' '${key}' (${message[key]})`);
+                    continue;
+                }
+            } catch (error) {
+                const message =
+                    `Publish '${parsedTopic.type}' '${key}' to '${re.name}' failed: '${error}'`;
+                logger.error(message);
+                logger.debug(error.stack);
+                this.legacyLog({type: `zigbee_publish_error`, message, meta: {friendly_name: re.name}});
+            }
+
+            usedConverters[endpointOrGroupID].push(converter);
+        }
+
+        for (const [ID, payload] of Object.entries(toPublish)) {
+            if (Object.keys(payload).length != 0) {
+                this.publishEntityState(toPublishEntity[ID], payload);
+            }
+        }
+
+        const scenesChanged = Object.values(usedConverters)
+            .some((cl) => cl.some((c) => c.key.some((k) => sceneConverterKeys.includes(k))));
+        if (scenesChanged) {
+            this.eventBus.emitScenesChanged();
+        }
+    }
+}

lib/extension/receive.js

@@ -1,181 +0,0 @@
-const settings = require('../util/settings');
-const logger = require('../util/logger');
-const utils = require('../util/utils');
-const debounce = require('debounce');
-const Extension = require('./extension');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-class Receive extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.elapsed = {};
-        this.debouncers = {};
-        this.eventBus.on('publishEntityState', (data) => this.onPublishEntityState(data));
-    }
-
-    async onZigbeeStarted() {
-        this.coordinator = this.zigbee.getDevicesByType('Coordinator')[0];
-    }
-
-    async onPublishEntityState(data) {
-        /**
-         * Prevent that outdated properties are being published.
-         * In case that e.g. the state is currently held back by a debounce and a new state is published
-         * remove it from the to be send debounced message.
-         */
-        if (data.entity.type === 'device' && this.debouncers[data.entity.device.ieeeAddr] &&
-            data.stateChangeReason !== 'publishDebounce') {
-            for (const key of Object.keys(data.payload)) {
-                delete this.debouncers[data.entity.device.ieeeAddr].payload[key];
-            }
-        }
-    }
-
-    publishDebounce(ieeeAddr, payload, time, debounceIgnore) {
-        if (!this.debouncers[ieeeAddr]) {
-            this.debouncers[ieeeAddr] = {
-                payload: {},
-                publish: debounce(() => {
-                    this.publishEntityState(ieeeAddr, this.debouncers[ieeeAddr].payload, 'publishDebounce');
-                    this.debouncers[ieeeAddr].payload = {};
-                }, time * 1000),
-            };
-        }
-
-        if (this.isPayloadConflicted(payload, this.debouncers[ieeeAddr].payload, debounceIgnore)) {
-            // publish previous payload immediately
-            this.debouncers[ieeeAddr].publish.flush();
-        }
-
-        // extend debounced payload with current
-        this.debouncers[ieeeAddr].payload = {...this.debouncers[ieeeAddr].payload, ...payload};
-        this.debouncers[ieeeAddr].publish();
-    }
-
-    // if debounce_ignore are specified (Array of strings)
-    // then all newPayload values with key present in debounce_ignore
-    // should equal or be undefined in oldPayload
-    // otherwise payload is conflicted
-    isPayloadConflicted(newPayload, oldPayload, debounceIgnore) {
-        let result = false;
-        Object.keys(oldPayload)
-            .filter((key) => (debounceIgnore || []).includes(key))
-            .forEach((key) => {
-                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
-                    result = true;
-                }
-            });
-
-        return result;
-    }
-
-    shouldProcess(type, data, resolvedEntity) {
-        if (type !== 'message' || !resolvedEntity) {
-            return false;
-        }
-
-        if (data.device.ieeeAddr === this.coordinator.ieeeAddr) {
-            logger.debug('Ignoring message from coordinator');
-            return false;
-        }
-
-        if (!resolvedEntity.definition) {
-            if (data.device.interviewing) {
-                logger.debug(`Skipping message, definition is undefined and still interviewing`);
-            } else {
-                logger.warn(`Received message from unsupported device with Zigbee model '${data.device.modelID}'`);
-                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
-            }
-
-            return false;
-        }
-
-        return true;
-    }
-
-    onZigbeeEvent(type, data, resolvedEntity) {
-        /**
-         * Handling of re-transmitted Xiaomi messages.
-         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
-         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
-         *
-         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
-         * The network address of these message is set to the one of the Xiaomi router.
-         * Therefore it looks like if the message came from the Xiaomi router, while in
-         * fact it came from the end device.
-         * Handling these message would result in false state updates.
-         * The group ID attribute of these message defines the network address of the end device.
-         */
-        if (type === 'message' && utils.isXiaomiDevice(data.device) && data.device.type === 'Router' && data.groupID) {
-            logger.debug('Handling re-transmitted Xiaomi message');
-            data.device = this.zigbee.getDeviceByNetworkAddress(data.groupID);
-            resolvedEntity = this.zigbee.resolveEntity(data.device);
-        }
-
-        if (!this.shouldProcess(type, data, resolvedEntity)) {
-            return;
-        }
-
-        const converters = resolvedEntity.definition.fromZigbee.filter((c) => {
-            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
-            return c.cluster === data.cluster && type;
-        });
-
-        // Check if there is an available converter, genOta messages are not interesting.
-        if (!converters.length && data.cluster !== 'genOta' && data.cluster !== 'genTime') {
-            logger.debug(
-                `No converter available for '${resolvedEntity.definition.model}' with cluster '${data.cluster}' ` +
-                `and type '${data.type}' and data '${stringify(data.data)}'`,
-            );
-            return;
-        }
-
-        // Convert this Zigbee message to a MQTT message.
-        // Get payload for the message.
-        // - If a payload is returned publish it to the MQTT broker
-        // - If NO payload is returned do nothing. This is for non-standard behaviour
-        //   for e.g. click switches where we need to count number of clicks and detect long presses.
-        const publish = (payload) => {
-            // Add device linkquality.
-            if (data.hasOwnProperty('linkquality')) {
-                payload.linkquality = data.linkquality;
-            }
-
-            if (settings.get().advanced.elapsed) {
-                const now = Date.now();
-                if (this.elapsed[data.device.ieeeAddr]) {
-                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
-                }
-
-                this.elapsed[data.device.ieeeAddr] = now;
-            }
-
-            // Check if we have to debounce
-            if (resolvedEntity.settings.debounce) {
-                this.publishDebounce(
-                    data.device.ieeeAddr, payload, resolvedEntity.settings.debounce,
-                    resolvedEntity.settings.debounce_ignore,
-                );
-            } else {
-                this.publishEntityState(data.device.ieeeAddr, payload);
-            }
-        };
-
-        const meta = {device: data.device, logger, state: this.state.get(data.device.ieeeAddr)};
-        let payload = {};
-        converters.forEach((converter) => {
-            const converted = converter.convert(
-                resolvedEntity.definition, data, publish, resolvedEntity.settings, meta,
-            );
-            if (converted) {
-                payload = {...payload, ...converted};
-            }
-        });
-
-        if (Object.keys(payload).length) {
-            publish(payload);
-        }
-    }
-}
-
-module.exports = Receive;

lib/extension/receive.ts

@@ -0,0 +1,167 @@
+import * as settings from '../util/settings';
+import logger from '../util/logger';
+import debounce from 'debounce';
+import Extension from './extension';
+import stringify from 'json-stable-stringify-without-jsonify';
+import bind from 'bind-decorator';
+
+type DebounceFunction = (() => void) & { clear(): void; } & { flush(): void; };
+
+export default class Receive extends Extension {
+    private elapsed: {[s: string]: number} = {};
+    private debouncers: {[s: string]: {payload: KeyValue, publish: DebounceFunction }} = {};
+
+    async start(): Promise<void> {
+        this.eventBus.onPublishEntityState(this, this.onPublishEntityState);
+        this.eventBus.onDeviceMessage(this, this.onDeviceMessage);
+    }
+
+    @bind async onPublishEntityState(data: eventdata.PublishEntityState): Promise<void> {
+        /**
+         * Prevent that outdated properties are being published.
+         * In case that e.g. the state is currently held back by a debounce and a new state is published
+         * remove it from the to be send debounced message.
+         */
+        if (data.entity.isDevice() && this.debouncers[data.entity.ieeeAddr] &&
+            data.stateChangeReason !== 'publishDebounce' && data.stateChangeReason !== 'lastSeenChanged') {
+            for (const key of Object.keys(data.message)) {
+                delete this.debouncers[data.entity.ieeeAddr].payload[key];
+            }
+        }
+    }
+
+    publishDebounce(device: Device, payload: KeyValue, time: number, debounceIgnore: string[]): void {
+        if (!this.debouncers[device.ieeeAddr]) {
+            this.debouncers[device.ieeeAddr] = {
+                payload: {},
+                publish: debounce(() => {
+                    this.publishEntityState(device, this.debouncers[device.ieeeAddr].payload, 'publishDebounce');
+                    this.debouncers[device.ieeeAddr].payload = {};
+                }, time * 1000),
+            };
+        }
+
+        if (this.isPayloadConflicted(payload, this.debouncers[device.ieeeAddr].payload, debounceIgnore)) {
+            // publish previous payload immediately
+            this.debouncers[device.ieeeAddr].publish.flush();
+        }
+
+        // extend debounced payload with current
+        this.debouncers[device.ieeeAddr].payload = {...this.debouncers[device.ieeeAddr].payload, ...payload};
+        this.debouncers[device.ieeeAddr].publish();
+    }
+
+    // if debounce_ignore are specified (Array of strings)
+    // then all newPayload values with key present in debounce_ignore
+    // should equal or be undefined in oldPayload
+    // otherwise payload is conflicted
+    isPayloadConflicted(newPayload: KeyValue, oldPayload: KeyValue, debounceIgnore: string[] | null): boolean {
+        let result = false;
+        Object.keys(oldPayload)
+            .filter((key) => (debounceIgnore || []).includes(key))
+            .forEach((key) => {
+                if (typeof newPayload[key] !== 'undefined' && newPayload[key] !== oldPayload[key]) {
+                    result = true;
+                }
+            });
+
+        return result;
+    }
+
+    shouldProcess(data: eventdata.DeviceMessage): boolean {
+        if (!data.device.definition) {
+            if (data.device.zh.interviewing) {
+                logger.debug(`Skipping message, definition is undefined and still interviewing`);
+            } else {
+                logger.warn(
+                    `Received message from unsupported device with Zigbee model '${data.device.zh.modelID}' ` +
+                    `and manufacturer name '${data.device.zh.manufacturerName}'`);
+                logger.warn(`Please see: https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html.`);
+            }
+
+            return false;
+        }
+
+        return true;
+    }
+
+    @bind async onDeviceMessage(data: eventdata.DeviceMessage): Promise<void> {
+        /* istanbul ignore next */
+        if (!data.device) return;
+
+        /**
+         * Handling of re-transmitted Xiaomi messages.
+         * https://github.com/Koenkk/zigbee2mqtt/issues/1238
+         * https://github.com/Koenkk/zigbee2mqtt/issues/3592
+         *
+         * Some Xiaomi router devices re-transmit messages from Xiaomi end devices.
+         * The network address of these message is set to the one of the Xiaomi router.
+         * Therefore it looks like if the message came from the Xiaomi router, while in
+         * fact it came from the end device.
+         * Handling these message would result in false state updates.
+         * The group ID attribute of these message defines the network address of the end device.
+         */
+        if (data.device.isXiaomi() && data.device.zh.type === 'Router' && data.groupID) {
+            logger.debug('Handling re-transmitted Xiaomi message');
+            data = {...data, device: this.zigbee.deviceByNetworkAddress(data.groupID)};
+        }
+
+        if (!this.shouldProcess(data)) return;
+
+        const converters = data.device.definition.fromZigbee.filter((c) => {
+            const type = Array.isArray(c.type) ? c.type.includes(data.type) : c.type === data.type;
+            return c.cluster === data.cluster && type;
+        });
+
+        // Check if there is an available converter, genOta messages are not interesting.
+        const ignoreClusters: (string | number)[] = ['genOta', 'genTime', 'genBasic'];
+        if (converters.length == 0 && !ignoreClusters.includes(data.cluster)) {
+            logger.debug(`No converter available for '${data.device.definition.model}' with ` +
+                `cluster '${data.cluster}' and type '${data.type}' and data '${stringify(data.data)}'`);
+            return;
+        }
+
+        // Convert this Zigbee message to a MQTT message.
+        // Get payload for the message.
+        // - If a payload is returned publish it to the MQTT broker
+        // - If NO payload is returned do nothing. This is for non-standard behaviour
+        //   for e.g. click switches where we need to count number of clicks and detect long presses.
+        const publish = (payload: KeyValue): void => {
+            if (settings.get().advanced.elapsed) {
+                const now = Date.now();
+                if (this.elapsed[data.device.ieeeAddr]) {
+                    payload.elapsed = now - this.elapsed[data.device.ieeeAddr];
+                }
+
+                this.elapsed[data.device.ieeeAddr] = now;
+            }
+
+            // Check if we have to debounce
+            if (data.device.settings.debounce) {
+                this.publishDebounce(data.device, payload, data.device.settings.debounce,
+                    data.device.settings.debounce_ignore);
+            } else {
+                this.publishEntityState(data.device, payload);
+            }
+        };
+
+        const meta = {device: data.device.zh, logger, state: this.state.get(data.device)};
+        let payload: KeyValue = {};
+        for (const converter of converters) {
+            try {
+                const converted = await converter.convert(
+                    data.device.definition, data, publish, data.device.settings, meta);
+                if (converted) {
+                    payload = {...payload, ...converted};
+                }
+            } catch (error) {
+                // istanbul ignore next
+                logger.error(`Exception while calling fromZigbee converter: ${error.message}}`);
+            }
+        }
+
+        if (Object.keys(payload).length) {
+            publish(payload);
+        }
+    }
+}

lib/extension/report.js

@@ -1,284 +0,0 @@
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const logger = require('../util/logger');
-const settings = require('../util/settings');
-const ZNLDP12LM = zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZNLDP12LM');
-const utils = require('../util/utils');
-const Extension = require('./extension');
-const debounce = require('debounce');
-const ZigbeeHerdsman = require('zigbee-herdsman');
-
-const defaultConfiguration = {
-    minimumReportInterval: 3, maximumReportInterval: 300, reportableChange: 1,
-};
-
-const devicesNotSupportingReporting = [
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'CC2530.ROUTER'),
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'BASICZBR3'),
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'ZM-CSW032-D'),
-    zigbeeHerdsmanConverters.devices.find((d) => d.model === 'TS0001'),
-];
-
-const reportKey = 1;
-
-const getColorCapabilities = async (endpoint) => {
-    if (endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities') === undefined) {
-        await endpoint.read('lightingColorCtrl', ['colorCapabilities']);
-    }
-
-    const value = endpoint.getClusterAttributeValue('lightingColorCtrl', 'colorCapabilities');
-    return {
-        colorTemperature: (value & 1<<4) > 0,
-        colorXY: (value & 1<<3) > 0,
-    };
-};
-
-const clusters = {
-    'genOnOff': [
-        {attribute: 'onOff', ...defaultConfiguration, minimumReportInterval: 0, reportableChange: 0},
-    ],
-    'genLevelCtrl': [
-        {attribute: 'currentLevel', ...defaultConfiguration},
-    ],
-    'lightingColorCtrl': [
-        {
-            attribute: 'colorTemperature', ...defaultConfiguration,
-            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorTemperature,
-        },
-        {
-            attribute: 'currentX', ...defaultConfiguration,
-            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
-        },
-        {
-            attribute: 'currentY', ...defaultConfiguration,
-            condition: async (endpoint) => (await getColorCapabilities(endpoint)).colorXY,
-        },
-    ],
-    'closuresWindowCovering': [
-        {attribute: 'currentPositionLiftPercentage', ...defaultConfiguration},
-        {attribute: 'currentPositionTiltPercentage', ...defaultConfiguration},
-    ],
-};
-
-const pollOnMessage = [
-    {
-        // Key is used this.pollDebouncers uniqueness
-        key: 1,
-        // On messages that have the cluster and type of below
-        cluster: {
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 2}},
-                {type: 'commandHueNotification', data: {button: 3}},
-            ],
-            genLevelCtrl: [
-                {type: 'commandStep', data: {}},
-                {type: 'commandStepWithOnOff', data: {}},
-                {type: 'commandStop', data: {}},
-                {type: 'commandMoveWithOnOff', data: {}},
-                {type: 'commandStopWithOnOff', data: {}},
-                {type: 'commandMove', data: {}},
-            ],
-        },
-        // Read the following attributes
-        read: {cluster: 'genLevelCtrl', attributes: ['currentLevel']},
-        // When the bound devices/members of group have the following manufacturerID
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-    },
-    {
-        key: 2,
-        cluster: {
-            genOnOff: [
-                {type: 'commandOn', data: {}},
-                {type: 'commandOff', data: {}},
-                {type: 'commandOffWithEffect', data: {}},
-            ],
-            manuSpecificPhilips: [
-                {type: 'commandHueNotification', data: {button: 1}},
-                {type: 'commandHueNotification', data: {button: 4}},
-            ],
-        },
-        read: {cluster: 'genOnOff', attributes: ['onOff']},
-        manufacturerID: ZigbeeHerdsman.Zcl.ManufacturerCode.Philips,
-    },
-];
-
-class Report extends Extension {
-    constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
-        super(zigbee, mqtt, state, publishEntityState, eventBus);
-        this.queue = new Set();
-        this.failed = new Set();
-        this.pollDebouncers = {};
-        this.enabled = settings.get().advanced.report;
-    }
-
-    shouldIgnoreClusterForDevice(cluster, definition) {
-        if (definition === ZNLDP12LM && cluster === 'closuresWindowCovering') {
-            // Device announces it but doesn't support it
-            // https://github.com/Koenkk/zigbee2mqtt/issues/2611
-            return true;
-        }
-
-        return false;
-    }
-
-    async setupReporting(resolvedEntity) {
-        const {device, definition} = resolvedEntity;
-
-        if (this.queue.has(device.ieeeAddr) || this.failed.has(device.ieeeAddr)) return;
-        this.queue.add(device.ieeeAddr);
-
-        const term1 = this.enabled ? 'Setup' : 'Disable';
-        const term2 = this.enabled ? 'setup' : 'disabled';
-
-        try {
-            for (const ep of device.endpoints) {
-                for (const [cluster, configuration] of Object.entries(clusters)) {
-                    if (ep.supportsInputCluster(cluster) && !this.shouldIgnoreClusterForDevice(cluster, definition)) {
-                        logger.debug(`${term1} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`);
-
-                        const items = [];
-                        for (const entry of configuration) {
-                            if (!entry.hasOwnProperty('condition') || (await entry.condition(ep))) {
-                                const toAdd = {...entry};
-                                if (!this.enabled) toAdd.maximumReportInterval = 0xFFFF;
-                                items.push(toAdd);
-                                delete items[items.length - 1].condition;
-                            }
-                        }
-
-                        this.enabled ?
-                            await ep.bind(cluster, this.coordinatorEndpoint) :
-                            await ep.unbind(cluster, this.coordinatorEndpoint);
-
-                        await ep.configureReporting(cluster, items);
-                        logger.info(
-                            `Successfully ${term2} reporting for '${device.ieeeAddr}' - ${ep.ID} - ${cluster}`,
-                        );
-                    }
-                }
-            }
-
-            if (this.enabled) {
-                device.meta.reporting = reportKey;
-            } else {
-                delete device.meta.reporting;
-                this.eventBus.emit('reportingDisabled', {device});
-            }
-        } catch (error) {
-            logger.error(
-                `Failed to ${term1.toLowerCase()} reporting for '${device.ieeeAddr}' - ${error.stack}`,
-            );
-
-            this.failed.add(device.ieeeAddr);
-        }
-
-        device.save();
-        this.queue.delete(device.ieeeAddr);
-    }
-
-    shouldSetupReporting(resolvedEntity, messageType) {
-        if (!resolvedEntity || !resolvedEntity.device || !resolvedEntity.definition ||
-            messageType === 'deviceLeave') return false;
-
-        const {device, definition} = resolvedEntity;
-        // Handle messages of type endDeviceAnnce and devIncoming.
-        // This message is typically send when a device comes online after being powered off
-        // Ikea TRADFRI tend to forget their reporting after powered off.
-        // Re-setup reporting.
-        // https://github.com/Koenkk/zigbee2mqtt/issues/966
-        if (this.enabled && messageType === 'deviceAnnounce' && utils.isIkeaTradfriDevice(device)) return true;
-
-        // These do not support reproting.
-        // https://github.com/Koenkk/zigbee-herdsman/issues/110
-        const philipsIgnoreSw = ['5.127.1.26581', '5.130.1.30000'];
-        if (device.manufacturerName === 'Philips' && philipsIgnoreSw.includes(device.softwareBuildID)) return false;
-
-        if (resolvedEntity.device.interviewing === true) return false;
-        if (device.type !== 'Router' || device.powerSource === 'Battery') return false;
-        // Gledopto devices don't support reporting.
-        if (devicesNotSupportingReporting.includes(definition) || definition.vendor === 'Gledopto') return false;
-
-        if (this.enabled && device.meta.hasOwnProperty('reporting') && device.meta.reporting === reportKey) {
-            return false;
-        }
-
-        if (!this.enabled && !device.meta.hasOwnProperty('reporting')) {
-            return false;
-        }
-
-        return true;
-    }
-
-    async onZigbeeStarted() {
-        this.coordinatorEndpoint = this.zigbee.getDevicesByType('Coordinator')[0].getEndpoint(1);
-
-        for (const device of this.zigbee.getClients()) {
-            const resolvedEntity = this.zigbee.resolveEntity(device);
-            if (this.shouldSetupReporting(resolvedEntity, null)) {
-                await this.setupReporting(resolvedEntity);
-            }
-        }
-    }
-
-    async onZigbeeEvent(type, data, resolvedEntity) {
-        if (this.shouldSetupReporting(resolvedEntity, type)) {
-            await this.setupReporting(resolvedEntity);
-        }
-
-        if (type === 'message') {
-            this.poll(data);
-        }
-    }
-
-    poll(message) {
-        /**
-         * This method poll bound endpoints and group members for state changes.
-         *
-         * A use case is e.g. a Hue Dimmer switch bound to a Hue bulb.
-         * Hue bulbs only report their on/off state.
-         * When dimming the bulb via the dimmer switch the state is therefore not reported.
-         * When we receive a message from a Hue dimmer we read the brightness from the bulb (if bound).
-         */
-        const polls = pollOnMessage.filter((p) =>
-            p.cluster[message.cluster] && p.cluster[message.cluster].find((c) => c.type === message.type &&
-            utils.equalsPartial(message.data, c.data)),
-        );
-
-        if (polls.length) {
-            let toPoll = [];
-
-            // Add bound devices
-            toPoll = toPoll.concat([].concat(...message.device.endpoints.map((e) =>
-                e.binds.map((e) => e).filter((e) => e.target))));
-            toPoll = toPoll.filter((e) => e.target.constructor.name === 'Endpoint');
-            toPoll = toPoll.filter((e) => e.target.getDevice().type !== 'Coordinator');
-            toPoll = toPoll.map((e) => e.target);
-
-            // If message is published to a group, add members of the group
-            const group = message.groupID !== 0 ? this.zigbee.getGroupByID(message.groupID) : null;
-            if (group) {
-                toPoll = toPoll.concat(group.members);
-            }
-
-            toPoll = new Set(toPoll);
-
-            for (const endpoint of toPoll) {
-                for (const poll of polls) {
-                    if (poll.manufacturerID !== endpoint.getDevice().manufacturerID) {
-                        continue;
-                    }
-
-                    const key = `${endpoint.deviceIeeeAddress}_${endpoint.ID}_${poll.key}`;
-                    if (!this.pollDebouncers[key]) {
-                        this.pollDebouncers[key] = debounce(async () => {
-                            await endpoint.read(poll.read.cluster, poll.read.attributes);
-                        }, 1000);
-                    }
-
-                    this.pollDebouncers[key]();
-                }
-            }
-        }
-    }
-}
-
-module.exports = Report;

lib/model/device.ts

@@ -0,0 +1,73 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+
+export default class Device {
+    public zh: zh.Device;
+    private _definition: zhc.Definition;
+
+    get ieeeAddr(): string {return this.zh.ieeeAddr;}
+    get ID(): string {return this.zh.ieeeAddr;}
+    get settings(): DeviceSettings {return {...settings.get().device_options, ...settings.getDevice(this.ieeeAddr)};}
+    get name(): string {
+        return this.zh.type === 'Coordinator' ? 'Coordinator' : this.settings?.friendly_name || this.ieeeAddr;
+    }
+    get definition(): zhc.Definition {
+        if (!this._definition && !this.zh.interviewing) {
+            this._definition = zigbeeHerdsmanConverters.findByDevice(this.zh);
+        }
+        return this._definition;
+    }
+
+    constructor(device: zh.Device) {
+        this.zh = device;
+    }
+
+    ensureInSettings(): void {
+        if (this.zh.type !== 'Coordinator' && !settings.getDevice(this.zh.ieeeAddr)) {
+            settings.addDevice(this.zh.ieeeAddr);
+        }
+    }
+
+    endpoint(key?: string | number): zh.Endpoint {
+        let endpoint: zh.Endpoint;
+        if (key == null || key == '') key = 'default';
+
+        if (!isNaN(Number(key))) {
+            endpoint = this.zh.getEndpoint(Number(key));
+        } else if (this.definition?.endpoint) {
+            const ID = this.definition?.endpoint?.(this.zh)[key];
+            if (ID) endpoint = this.zh.getEndpoint(ID);
+            else if (key === 'default') endpoint = this.zh.endpoints[0];
+            else return null;
+        } else {
+            /* istanbul ignore next */
+            if (key !== 'default') return null;
+            endpoint = this.zh.endpoints[0];
+        }
+
+        return endpoint;
+    }
+
+    endpointName(endpoint: zh.Endpoint): string {
+        let name = null;
+        if (this.definition?.endpoint) {
+            name = Object.entries(this.definition?.endpoint(this.zh)).find((e) => e[1] == endpoint.ID)[0];
+        }
+        /* istanbul ignore next */
+        return name === 'default' ? null : name;
+    }
+
+    isXiaomi(): boolean {
+        const xiaomiManufacturerID = [4151, 4447];
+        /* istanbul ignore next */
+        return this.zh.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(this.zh.manufacturerID) &&
+            (!this.zh.manufacturerName || !this.zh.manufacturerName.startsWith('Trust'));
+    }
+
+    isIkeaTradfri(): boolean {return this.zh.manufacturerID === 4476;}
+
+    isDevice(): this is Device {return true;}
+    /* istanbul ignore next */
+    isGroup(): this is Group {return false;}
+}

lib/model/group.ts

@@ -0,0 +1,23 @@
+/* eslint-disable brace-style */
+import * as settings from '../util/settings';
+import zigbeeHerdsmanConverters from 'zigbee-herdsman-converters';
+
+export default class Group {
+    public zh: zh.Group;
+
+    get ID(): number {return this.zh.groupID;}
+    get settings(): GroupSettings {return settings.getGroup(this.ID);}
+    get name(): string {return this.settings?.friendly_name || this.ID.toString();}
+
+    constructor(group: zh.Group) {
+        this.zh = group;
+    }
+
+    membersDefinitions(): zhc.Definition[] {
+        return this.zh.members.map((m) =>
+            zigbeeHerdsmanConverters.findByDevice(m.getDevice())).filter((d) => d) as zhc.Definition[];
+    }
+
+    isDevice(): this is Device {return false;}
+    isGroup(): this is Group {return true;}
+}

lib/mqtt.ts

@@ -1,24 +1,30 @@
-const mqtt = require('mqtt');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const fs = require('fs');
-const events = require('events');
+import mqtt from 'mqtt';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import utils from './util/utils';
+import fs from 'fs';
+import bind from 'bind-decorator';
 
-class MQTT extends events.EventEmitter {
-    constructor() {
-        super();
-        this.onMessage = this.onMessage.bind(this);
+export default class MQTT {
+    private publishedTopics: Set<string> = new Set();
+    private connectionTimer: NodeJS.Timeout;
+    private client: mqtt.MqttClient;
+    private eventBus: EventBus;
+
+    constructor(eventBus: EventBus) {
+        this.eventBus = eventBus;
     }
 
-    async connect() {
+    async connect(): Promise<void> {
         const mqttSettings = settings.get().mqtt;
         logger.info(`Connecting to MQTT server at ${mqttSettings.server}`);
 
-        const options = {
+        const options: mqtt.IClientOptions = {
             will: {
                 topic: `${settings.get().mqtt.base_topic}/bridge/state`,
                 payload: 'offline',
-                retain: true,
+                retain: settings.get().mqtt.force_disable_retain ? false : true,
+                qos: 1,
             },
         };
 
@@ -44,8 +50,11 @@ class MQTT extends events.EventEmitter {
         }
 
         if (mqttSettings.user && mqttSettings.password) {
+            logger.debug(`Using MQTT login with username: ${mqttSettings.user}`);
             options.username = mqttSettings.user;
             options.password = mqttSettings.password;
+        } else {
+            logger.debug(`Using MQTT anonymous login`);
         }
 
         if (mqttSettings.client_id) {
@@ -58,49 +67,68 @@ class MQTT extends events.EventEmitter {
             options.rejectUnauthorized = false;
         }
 
-        // Set timer at interval to check if connected to MQTT server.
-        const interval = 10 * 1000; // seconds * 1000.
-        this.connectionTimer = setInterval(() => {
-            if (this.client.reconnecting) {
-                logger.error('Not connected to MQTT server!');
-            }
-        }, interval);
-
-        return new Promise((resolve) => {
+        return new Promise((resolve, reject) => {
             this.client = mqtt.connect(mqttSettings.server, options);
 
-            this.client.on('connect', () => {
-                logger.info('Connected to MQTT server');
+            const onConnect = this.onConnect;
+            this.client.on('connect', async () => {
+                await onConnect();
                 resolve();
             });
 
+            this.client.on('error', (err) => reject(err));
             this.client.on('message', this.onMessage);
         });
     }
 
-    async disconnect() {
+    @bind private async onConnect(): Promise<void> {
+        // Set timer at interval to check if connected to MQTT server.
+        clearTimeout(this.connectionTimer);
+        this.connectionTimer = setInterval(() => {
+            if (this.client.reconnecting) {
+                logger.error('Not connected to MQTT server!');
+            }
+        }, utils.seconds(10));
+
+        logger.info('Connected to MQTT server');
+        this.subscribe(`${settings.get().mqtt.base_topic}/#`);
+        await this.publish('bridge/state', 'online', {retain: true, qos: 0});
+    }
+
+    async disconnect(): Promise<void> {
         clearTimeout(this.connectionTimer);
-        this.connectionTimer = null;
         await this.publish('bridge/state', 'offline', {retain: true, qos: 0});
         logger.info('Disconnecting from MQTT server');
         this.client.end();
     }
 
-    subscribe(topic) {
+    subscribe(topic: string): void {
         this.client.subscribe(topic);
     }
 
-    onMessage(topic, message) {
-        this.emit('message', {topic, message: message + ''});
+    @bind public onMessage(topic: string, message: string): void {
+        // Since we subscribe to zigbee2mqtt/# we also receive the message we send ourselves, skip these.
+        if (!this.publishedTopics.has(topic)) {
+            logger.debug(`Received MQTT message on '${topic}' with data '${message}'`);
+            this.eventBus.emitMQTTMessage({topic, message: message + ''});
+        }
     }
 
-    isConnected() {
+    isConnected(): boolean {
         return this.client && !this.client.reconnecting;
     }
 
-    async publish(topic, payload, options, base=settings.get().mqtt.base_topic, skipLog=false) {
+    async publish(topic: string, payload: string, options: MQTTOptions={},
+        base=settings.get().mqtt.base_topic, skipLog=false, skipReceive=true,
+    ): Promise<void> {
+        const defaultOptions: {qos: mqtt.QoS, retain: boolean} = {qos: 0, retain: false};
         topic = `${base}/${topic}`;
-        options = {qos: 0, retain: false, ...options};
+
+        if (skipReceive) {
+            this.publishedTopics.add(topic);
+        }
+
+        this.eventBus.emitMQTTMessagePublished({topic, payload, options: {...defaultOptions, ...options}});
 
         if (!this.isConnected()) {
             if (!skipLog) {
@@ -114,13 +142,13 @@ class MQTT extends events.EventEmitter {
             logger.info(`MQTT publish: topic '${topic}', payload '${payload}'`);
         }
 
+        const actualOptions: mqtt.IClientPublishOptions = {...defaultOptions, ...options};
+        if (settings.get().mqtt.force_disable_retain) {
+            actualOptions.retain = false;
+        }
+
         return new Promise((resolve) => {
-            this.client.publish(topic, payload, options, () => {
-                this.emit('publishedMessage', {topic, payload, options});
-                resolve();
-            });
+            this.client.publish(topic, payload, actualOptions, () => resolve());
         });
     }
 }
-
-module.exports = MQTT;

lib/state.js

@@ -1,120 +0,0 @@
-const logger = require('./util/logger');
-const data = require('./util/data');
-const settings = require('./util/settings');
-const fs = require('fs');
-const objectAssignDeep = require('object-assign-deep');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const saveInterval = 1000 * 60 * 5; // 5 minutes
-
-const dontCacheProperties = [
-    'action', 'action_.*', 'button', 'button_left', 'button_right', 'click', 'forgotten', 'keyerror',
-    'step_size', 'transition_time', 'group_list', 'group_capacity', 'no_occupancy_since',
-    'step_mode', 'transition_time', 'duration', 'elapsed', 'from_side', 'to_side',
-];
-
-class State {
-    constructor(eventBus) {
-        this.state = {};
-        this.file = data.joinPath('state.json');
-        this.timer = null;
-        this.eventBus = eventBus;
-    }
-
-    start() {
-        this._load();
-
-        // Save the state on every interval
-        this.clearTimer();
-        this.timer = setInterval(() => this.save(), saveInterval);
-    }
-
-    clearTimer() {
-        if (this.timer) {
-            clearTimeout(this.timer);
-            this.timer = null;
-        }
-    }
-
-    stop() {
-        this.clearTimer();
-        this.save();
-    }
-
-    _load() {
-        if (fs.existsSync(this.file)) {
-            try {
-                this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
-                logger.debug(`Loaded state from file ${this.file}`);
-            } catch (e) {
-                logger.debug(`Failed to load state from file ${this.file} (corrupt file?)`);
-            }
-        } else {
-            logger.debug(`Can't load state from file ${this.file} (doesn't exist)`);
-        }
-    }
-
-    save() {
-        if (settings.get().advanced.cache_state_persistent) {
-            logger.debug(`Saving state to file ${this.file}`);
-            const json = stringify(this.state, null, 4);
-            try {
-                fs.writeFileSync(this.file, json, 'utf8');
-            } catch (e) {
-                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
-            }
-        } else {
-            logger.debug(`Not saving state`);
-        }
-    }
-
-    exists(ID) {
-        return this.state.hasOwnProperty(ID);
-    }
-
-    get(ID) {
-        return this.state[ID];
-    }
-
-    set(ID, state, reason=null) {
-        const toState = objectAssignDeep.noMutate(state);
-
-        for (const property of Object.keys(toState)) {
-            if (dontCacheProperties.find((p) => property.match(p))) {
-                delete toState[property];
-            }
-        }
-
-        const fromState = this.state[ID];
-
-        this.state[ID] = toState;
-
-        this.eventBus.emit('stateChange', {ID, from: fromState, to: toState, reason});
-    }
-
-    removeKey(ID, path) {
-        if (this.exists(ID)) {
-            let state = this.state[ID];
-            for (let i = 0; i < path.length; i++) {
-                const key = path[i];
-                if (i === path.length - 1) {
-                    delete state[key];
-                } else {
-                    if (state[key]) {
-                        state = state[key];
-                    } else {
-                        break;
-                    }
-                }
-            }
-        }
-    }
-
-    remove(ID) {
-        if (this.exists(ID)) {
-            delete this.state[ID];
-        }
-    }
-}
-
-module.exports = State;

lib/state.ts

@@ -0,0 +1,95 @@
+import logger from './util/logger';
+import data from './util/data';
+import * as settings from './util/settings';
+import fs from 'fs';
+import objectAssignDeep from 'object-assign-deep';
+
+const saveInterval = 1000 * 60 * 5; // 5 minutes
+
+const dontCacheProperties = [
+    '^action$', '^action_.*$', '^button$', '^button_left$', '^button_right$', '^click$', '^forgotten$', '^keyerror$',
+    '^step_size$', '^transition_time$', '^group_list$', '^group_capacity$', '^no_occupancy_since$',
+    '^step_mode$', '^transition_time$', '^duration$', '^elapsed$', '^from_side$', '^to_side$',
+];
+
+class State {
+    private state: {[s: string | number]: KeyValue} = {};
+    private file = data.joinPath('state.json');
+    private timer: NodeJS.Timer = null;
+    private eventBus: EventBus;
+
+    constructor(eventBus: EventBus) {
+        this.eventBus = eventBus;
+    }
+
+    start(): void {
+        this.load();
+
+        // Save the state on every interval
+        this.timer = setInterval(() => this.save(), saveInterval);
+        this.eventBus.onDeviceLeave(this, (data) => delete this.state[data.ieeeAddr]);
+    }
+
+    stop(): void {
+        this.eventBus.removeListeners(this);
+        clearTimeout(this.timer);
+        this.save();
+    }
+
+    private load(): void {
+        if (fs.existsSync(this.file)) {
+            try {
+                this.state = JSON.parse(fs.readFileSync(this.file, 'utf8'));
+                logger.debug(`Loaded state from file ${this.file}`);
+            } catch (e) {
+                logger.debug(`Failed to load state from file ${this.file} (corrupt file?)`);
+            }
+        } else {
+            logger.debug(`Can't load state from file ${this.file} (doesn't exist)`);
+        }
+    }
+
+    private save(): void {
+        if (settings.get().advanced.cache_state_persistent) {
+            logger.debug(`Saving state to file ${this.file}`);
+            const json = JSON.stringify(this.state, null, 4);
+            try {
+                fs.writeFileSync(this.file, json, 'utf8');
+            } catch (e) {
+                logger.error(`Failed to write state to '${this.file}' (${e.message})`);
+            }
+        } else {
+            logger.debug(`Not saving state`);
+        }
+    }
+
+    exists(entity: Device | Group): boolean {
+        return this.state.hasOwnProperty(entity.ID);
+    }
+
+    get(entity: Group | Device): KeyValue {
+        return this.state[entity.ID];
+    }
+
+    set(entity: Group | Device, update: KeyValue, reason: string=null): KeyValue {
+        const fromState = this.state[entity.ID] || {};
+        const toState = objectAssignDeep({}, fromState, update);
+        const result = {...toState};
+
+        for (const property of Object.keys(toState)) {
+            if (dontCacheProperties.find((p) => property.match(p))) {
+                delete toState[property];
+            }
+        }
+
+        this.state[entity.ID] = toState;
+        this.eventBus.emitStateChange({entity, from: fromState, to: toState, reason, update});
+        return result;
+    }
+
+    remove(ID: string | number): void {
+        delete this.state[ID];
+    }
+}
+
+export default State;

lib/types/json-stable-stringify-without-jsonify.d.ts

@@ -0,0 +1,3 @@
+declare module 'json-stable-stringify-without-jsonify' {
+    export default function (obj: unknown): string;
+}

lib/types/mkdir-recursive.ts

@@ -0,0 +1,3 @@
+declare module 'mkdir-recursive' {
+    export function mkdirSync(path: string): void;
+}

lib/types/types.d.ts

@@ -0,0 +1,298 @@
+/* eslint-disable camelcase */
+import type {
+    Device as ZHDevice,
+    Group as ZHGroup,
+    Endpoint as ZHEndpoint,
+} from 'zigbee-herdsman/dist/controller/model';
+
+import type {
+    NetworkParameters as ZHNetworkParameters,
+    CoordinatorVersion as ZHCoordinatorVersion,
+    LQI as ZHLQI,
+    RoutingTable as ZHRoutingTable,
+    RoutingTableEntry as ZHRoutingTableEntry,
+} from 'zigbee-herdsman/dist/adapter/tstype';
+
+import type {
+    Cluster as ZHCluster,
+} from 'zigbee-herdsman/dist/zcl/tstype';
+
+import type * as ZHEvents from 'zigbee-herdsman/dist/controller/events';
+
+import type TypeEventBus from 'lib/eventBus';
+import type TypeMQTT from 'lib/mqtt';
+import type TypeState from 'lib/state';
+import type TypeZigbee from 'lib/zigbee';
+import type TypeDevice from 'lib/model/device';
+import type TypeGroup from 'lib/model/group';
+import type TypeExtension from 'lib/extension/extension';
+
+import type mqtt from 'mqtt';
+
+declare global {
+    // Define some class types as global
+    type EventBus = TypeEventBus;
+    type MQTT = TypeMQTT;
+    type Zigbee = TypeZigbee;
+    type Group = TypeGroup;
+    type Device = TypeDevice;
+    type State = TypeState;
+    type Extension = TypeExtension;
+
+    // Types
+    interface MQTTResponse {data: KeyValue, status: 'error' | 'ok', error?: string, transaction?: string}
+    interface MQTTOptions {qos?: mqtt.QoS, retain?: boolean, properties?: {messageExpiryInterval: number}}
+    type StateChangeReason = 'publishDebounce' | 'groupOptimistic' | 'lastSeenChanged';
+    type PublishEntityState = (entity: Device | Group, payload: KeyValue,
+        stateChangeReason?: StateChangeReason) => Promise<void>;
+    type RecursivePartial<T> = {[P in keyof T]?: RecursivePartial<T[P]>;};
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    interface KeyValue {[s: string]: any}
+
+    // zigbee-herdsman
+    namespace zh {
+        type Endpoint = ZHEndpoint;
+        type Device = ZHDevice;
+        type Group = ZHGroup;
+        type LQI = ZHLQI;
+        type RoutingTable = ZHRoutingTable;
+        type RoutingTableEntry = ZHRoutingTableEntry;
+        type CoordinatorVersion = ZHCoordinatorVersion;
+        type NetworkParameters = ZHNetworkParameters;
+        type Cluster = ZHCluster;
+        interface Bind {
+            cluster: zh.Cluster;
+            target: zh.Endpoint | zh.Group;
+        }
+    }
+
+    // zigbee-herdsman-converters
+    namespace zhc {
+        interface Logger {
+            info: (message: string) => void;
+            warn: (message: string) => void;
+            error: (message: string) => void;
+            debug: (message: string) => void;
+        }
+
+        interface ToZigbeeConverterGetMeta {message?: KeyValue, mapped?: Definition | Definition[]}
+
+        interface ToZigbeeConverterResult {state: KeyValue,
+            membersState: {[s: string]: KeyValue}, readAfterWriteTime?: number}
+
+        interface ToZigbeeConverter {
+            key: string[],
+            convertGet?: (entity: zh.Endpoint | zh.Group, key: string, meta: ToZigbeeConverterGetMeta) => Promise<void>
+            convertSet?: (entity: zh.Endpoint | zh.Group, key: string, value: KeyValue | string | number,
+                meta: {state: KeyValue}) => Promise<ToZigbeeConverterResult>
+        }
+
+        interface FromZigbeeConverter {
+            cluster: string,
+            type: string[] | string,
+            convert: (model: Definition, msg: KeyValue, publish: (payload: KeyValue) => void, options: KeyValue,
+                meta: {state: KeyValue, logger: Logger, device: zh.Device}) => Promise<KeyValue>,
+        }
+
+        interface DefinitionExposeFeature {name: string, endpoint?: string,
+            property: string, value_max?: number, value_min?: number,
+            value_off?: string, value_on?: string, value_step?: number, values: string[], access: number}
+
+        interface DefinitionExpose {
+            type: string, name?: string, features?: DefinitionExposeFeature[],
+            endpoint?: string, values?: string[], value_off?: string, value_on?: string,
+            access: number, property: string, unit?: string,
+            value_min?: number, value_max?: number}
+
+        interface Definition {
+            model: string,
+            zigbeeModel: string[],
+            endpoint?: (device: zh.Device) => {[s: string]: number}
+            toZigbee: ToZigbeeConverter[]
+            fromZigbee: FromZigbeeConverter[]
+            icon?: string
+            description: string
+            options: zhc.DefinitionExpose[],
+            vendor: string
+            exposes: DefinitionExpose[]
+            configure?: (device: zh.Device, coordinatorEndpoint: zh.Endpoint, logger: Logger) => Promise<void>;
+            onEvent?: (type: string, data: KeyValue, device: zh.Device, settings: KeyValue) => Promise<void>;
+            ota?: {
+                isUpdateAvailable: (device: zh.Device, logger: Logger, data?: KeyValue) => Promise<boolean>;
+                updateToLatest: (device: zh.Device, logger: Logger,
+                    onProgress: (progress: number, remaining: number) => void) => Promise<void>;
+            }
+        }
+
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        type ExternalDefinition = Definition & {homeassistant: any};
+    }
+
+    namespace eventdata {
+        type DeviceRenamed = { device: Device, homeAssisantRename: boolean, from: string, to: string };
+        type DeviceRemoved = { ieeeAddr: string, name: string };
+        type MQTTMessage = { topic: string, message: string };
+        type MQTTMessagePublished = { topic: string, payload: string, options: {retain: boolean, qos: number} };
+        type StateChange = {
+            entity: Device | Group, from: KeyValue, to: KeyValue, reason: string | null, update: KeyValue };
+        type PermitJoinChanged = ZHEvents.PermitJoinChangedPayload;
+        type LastSeenChanged = { device: Device };
+        type DeviceNetworkAddressChanged = { device: Device };
+        type DeviceAnnounce = { device: Device };
+        type DeviceInterview = { device: Device, status: 'started' | 'successful' | 'failed' };
+        type DeviceJoined = { device: Device };
+        type ReportingDisabled = { device: Device };
+        type DeviceLeave = { ieeeAddr: string, name: string };
+        type GroupMembersChanged = {group: Group, action: 'remove' | 'add' | 'remove_all',
+            endpoint: zh.Endpoint, skipDisableReporting: boolean };
+        type PublishEntityState = {entity: Group | Device, message: KeyValue, stateChangeReason: StateChangeReason };
+        type DeviceMessage = {
+            type: ZHEvents.MessagePayloadType;
+            device: Device;
+            endpoint: zh.Endpoint;
+            linkquality: number;
+            groupID: number;
+            cluster: string | number;
+            data: KeyValue | Array<string | number>;
+            meta: {zclTransactionSequenceNumber?: number;};
+        };
+    }
+
+    // Settings
+    // eslint-disable camelcase
+    interface Settings {
+        homeassistant?: boolean,
+        devices?: {[s: string]: DeviceSettings},
+        groups?: {[s: string]: GroupSettings},
+        passlist: string[],
+        blocklist: string[],
+        whitelist: string[],
+        ban: string[],
+        availability?: boolean | {
+            active?: {timeout?: number},
+            passive?: {timeout?: number}
+        },
+        permit_join: boolean,
+        frontend?: {
+            auth_token?: string,
+            host?: string,
+            port?: number,
+            url?: string,
+        },
+        mqtt: {
+            base_topic: string,
+            include_device_information: boolean,
+            force_disable_retain: boolean
+            version?: number,
+            user?: string,
+            password?: string,
+            server: string,
+            ca?: string,
+            keepalive?: number,
+            key?: string,
+            cert?: string,
+            client_id?: string,
+            reject_unauthorized?: boolean,
+        },
+        serial: {
+            disable_led?: boolean,
+            port?: string,
+            adapter?: 'deconz' | 'zstack' | 'ezsp' | 'zigate'
+        },
+        device_options: KeyValue,
+        map_options: {
+            graphviz: {
+                colors: {
+                    fill: {
+                        enddevice: string,
+                        coordinator: string,
+                        router: string,
+                    },
+                    font: {
+                        coordinator: string,
+                        router: string,
+                        enddevice: string,
+                    },
+                    line: {
+                        active: string,
+                        inactive: string,
+                    },
+                },
+            },
+        },
+        experimental: {
+            output: 'json' | 'attribute' | 'attribute_and_json',
+            transmit_power?: number,
+        },
+        advanced: {
+            legacy_api: boolean,
+            log_rotation: boolean,
+            log_symlink_current: boolean,
+            log_output: ('console' | 'file' | 'syslog')[],
+            log_directory: string,
+            log_file: string,
+            log_level: 'debug' | 'info' | 'error' | 'warn',
+            log_syslog: KeyValue,
+            soft_reset_timeout: number,
+            pan_id: number | 'GENERATE',
+            ext_pan_id: number[],
+            channel: number,
+            adapter_concurrent: number | null,
+            adapter_delay: number | null,
+            cache_state: boolean,
+            cache_state_persistent: boolean,
+            cache_state_send_on_startup: boolean,
+            last_seen: 'disable' | 'ISO_8601' | 'ISO_8601_local' | 'epoch',
+            elapsed: boolean,
+            network_key: number[] | 'GENERATE',
+            report: boolean,
+            homeassistant_discovery_topic: string,
+            homeassistant_status_topic: string,
+            homeassistant_legacy_entity_attributes: boolean,
+            homeassistant_legacy_triggers: boolean,
+            timestamp_format: string,
+            baudrate?: number,
+            rtscts?: boolean,
+            ikea_ota_use_test_url?: boolean,
+            // below are deprecated
+            availability_timeout?: number,
+            availability_blocklist?: string[],
+            availability_passlist?: string[],
+            availability_blacklist?: string[],
+            availability_whitelist?: string[],
+        },
+        ota: {
+            update_check_interval: number,
+            disable_automatic_update_check: boolean
+        },
+        external_converters: string[],
+    }
+
+    interface DeviceSettings {
+        ID?: string,
+        retention?: number,
+        availability?: boolean | {timeout: number},
+        optimistic?: boolean,
+        retrieve_state?: boolean,
+        debounce?: number,
+        debounce_ignore?: string[],
+        filtered_optimistic?: string[],
+        icon?: string,
+        homeassistant?: KeyValue,
+        legacy?: boolean,
+        filtered_attributes?: string[],
+        friendly_name: string,
+    }
+
+    interface GroupSettings {
+        devices?: string[],
+        ID?: number,
+        optimistic?: boolean,
+        filtered_optimistic?: string[],
+        retrieve_state?: boolean,
+        homeassistant?: KeyValue,
+        filtered_attributes?: string[],
+        friendly_name: string,
+    }
+}
+

lib/types/zigbee-herdsman-converters.d.ts

@@ -0,0 +1,13 @@
+declare module 'zigbee-herdsman-converters' {
+    export function findByDevice(device: zh.Device): zhc.Definition;
+    export function getConfigureKey(definition: zhc.Definition): string | number;
+    export const toZigbeeConverters: {[s: string]: zhc.ToZigbeeConverter};
+    export const definitions: zhc.Definition[];
+    export function addDeviceDefinition(definition: zhc.Definition): Promise<void>;
+    export function onEvent(type: string, data: KeyValue, device: zh.Device): Promise<void>;
+}
+
+declare module 'zigbee-herdsman-converters/lib/ota/tradfri' {
+    export function useTestURL(): void;
+}
+

lib/types/zigbee2mqtt-frontend.ts

@@ -0,0 +1,3 @@
+declare module 'zigbee2mqtt-frontend' {
+    export function getPath(): string;
+}

lib/util/data.js

@@ -1,22 +0,0 @@
-const path = require('path');
-
-let dataPath = null;
-
-function load() {
-    if (process.env.ZIGBEE2MQTT_DATA) {
-        dataPath = process.env.ZIGBEE2MQTT_DATA;
-    } else {
-        dataPath = path.join(__dirname, '..', '..', 'data');
-        dataPath = path.normalize(dataPath);
-    }
-}
-
-load();
-
-module.exports = {
-    joinPath: (file) => path.join(dataPath, file),
-    getPath: () => dataPath,
-
-    // For test only.
-    _reload: () => load(),
-};

lib/util/data.ts

@@ -0,0 +1,29 @@
+import path from 'path';
+
+let dataPath: string = null;
+
+function load(): void {
+    if (process.env.ZIGBEE2MQTT_DATA) {
+        dataPath = process.env.ZIGBEE2MQTT_DATA;
+    } else {
+        dataPath = path.join(__dirname, '..', '..', 'data');
+        dataPath = path.normalize(dataPath);
+    }
+}
+
+load();
+
+function joinPath(file: string): string {
+    return path.join(dataPath, file);
+}
+
+function getPath(): string {
+    return dataPath;
+}
+
+// eslint-disable-next-line camelcase
+function __testingOnly_reload(): void {
+    load();
+}
+
+export default {joinPath, getPath, __testingOnly_reload};

lib/util/logger.js

@@ -1,140 +0,0 @@
-const winston = require('winston');
-const moment = require('moment');
-const settings = require('./settings');
-const path = require('path');
-const fs = require('fs');
-const fx = require('mkdir-recursive');
-const rimraf = require('rimraf');
-const colorizer = winston.format.colorize();
-const assert = require('assert');
-
-// What transports to enable
-const output = settings.get().advanced.log_output;
-
-// Directory to log to
-const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
-const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
-const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
-
-// Make sure that log directoy exsists when not logging to stdout only
-if (output.includes('file')) {
-    fx.mkdirSync(directory);
-}
-
-// Determine the log level.
-const level = settings.get().advanced.log_level;
-const validLevels = ['info', 'error', 'warn', 'debug'];
-assert(validLevels.includes(level), `'${level}' is not a valid log_level, use one of '${validLevels.join(', ')}'`);
-
-const levelWithCompensatedLength = {
-    'info': 'info ',
-    'error': 'error',
-    'warn': 'warn ',
-    'debug': 'debug',
-};
-
-/* istanbul ignore next */
-const timestampFormat = () => moment().format(settings.get().advanced.timestamp_format);
-
-// Setup default console logger
-const transportsToUse = [
-    new winston.transports.Console({
-        level,
-        silent: !output.includes('console'),
-        format: winston.format.combine(
-            winston.format.timestamp({format: timestampFormat}),
-            winston.format.printf(/* istanbul ignore next */(info) => {
-                let {timestamp, level, message} = info;
-                level = level === 'warning' ? 'warn' : level;
-                const prefix = colorizer.colorize(level, `Zigbee2MQTT:${levelWithCompensatedLength[level]}`);
-                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
-            }),
-        ),
-    }),
-];
-
-// Add file logger when enabled
-// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
-const transportFileOptions = {
-    filename: path.join(directory, logFilename),
-    json: false,
-    level,
-    format: winston.format.combine(
-        winston.format.timestamp({format: timestampFormat}),
-        winston.format.printf(/* istanbul ignore next */(info) => {
-            let {timestamp, level, message} = info;
-            level = level === 'warning' ? 'warn' : level;
-            return `${levelWithCompensatedLength[level]} ${timestamp.split('.')[0]}: ${message}`;
-        }),
-    ),
-};
-
-if (settings.get().advanced.log_rotation) {
-    transportFileOptions.tailable = true;
-    transportFileOptions.maxFiles = 3; // Keep last 3 files
-    transportFileOptions.maxsize = 10000000; // 10MB
-}
-
-if (output.includes('file')) {
-    transportsToUse.push(new winston.transports.File(transportFileOptions));
-}
-
-/* istanbul ignore next */
-if (output.includes('syslog')) {
-    require('winston-syslog').Syslog;
-    const options = {
-        app_name: 'Zigbee2MQTT',
-        format: winston.format.printf(/* istanbul ignore next */(info) => {
-            return `${info.message}`;
-        }),
-        ...settings.get().advanced.log_syslog,
-    };
-    if (options.hasOwnProperty('type')) options.type = options.type.toString();
-    transportsToUse.push(new winston.transports.Syslog(options));
-}
-
-// Create logger
-const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
-
-// Cleanup any old log directory.
-function cleanup() {
-    if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
-        const rootDirectory = path.join(directory, '..');
-
-        let directories = fs.readdirSync(rootDirectory).map((d) => {
-            d = path.join(rootDirectory, d);
-            return {path: d, birth: fs.statSync(d).mtime};
-        });
-
-        directories.sort((a, b) => b.birth - a.birth);
-        directories = directories.slice(10, directories.length);
-        directories.forEach((dir) => {
-            logger.debug(`Removing old log directory '${dir.path}'`);
-            rimraf.sync(dir.path);
-        });
-    }
-}
-
-logger.cleanup = cleanup;
-logger.getLevel = () => transportsToUse[0].level;
-logger.setLevel = (level) => {
-    transportsToUse.forEach((transport) => transport.level = level);
-};
-
-// Print to user what logging is enabled
-if (output.includes('file')) {
-    if (output.includes('console')) {
-        logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
-    } else {
-        logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
-    }
-    logger.cleanup();
-} else if (output.includes('console')) {
-    logger.info(`Logging to console only'`);
-}
-
-// winston.config.syslog.levels doesnt have warn, but is required for syslog.
-/* istanbul ignore next */
-logger.warn = (message) => logger.warning(message);
-
-module.exports = logger;

lib/util/logger.ts

@@ -0,0 +1,186 @@
+import winston from 'winston';
+import moment from 'moment';
+import * as settings from './settings';
+import path from 'path';
+import fs from 'fs';
+import fx from 'mkdir-recursive';
+import rimraf from 'rimraf';
+import assert from 'assert';
+
+const colorizer = winston.format.colorize();
+
+// What transports to enable
+const output = settings.get().advanced.log_output;
+
+// Directory to log to
+const timestamp = moment(Date.now()).format('YYYY-MM-DD.HH-mm-ss');
+const directory = settings.get().advanced.log_directory.replace('%TIMESTAMP%', timestamp);
+const logFilename = settings.get().advanced.log_file.replace('%TIMESTAMP%', timestamp);
+
+// Make sure that log directoy exsists when not logging to stdout only
+if (output.includes('file')) {
+    fx.mkdirSync(directory);
+
+    if (settings.get().advanced.log_symlink_current) {
+        const current = settings.get().advanced.log_directory.replace('%TIMESTAMP%', 'current');
+        const actual = './' + timestamp;
+        if (fs.existsSync(current)) {
+            fs.unlinkSync(current);
+        }
+        fs.symlinkSync(actual, current);
+    }
+}
+
+type Z2MLogLevel = 'warn' | 'debug' | 'info' | 'error';
+type WinstonLogLevel = 'warning' | 'debug' | 'info' | 'error';
+
+const z2mToWinstonLevel = (level: Z2MLogLevel): WinstonLogLevel => level === 'warn' ? 'warning' : level;
+const winstonToZ2mLevel = (level: WinstonLogLevel): Z2MLogLevel => level === 'warning' ? 'warn' : level;
+
+// Determine the log level.
+const z2mLevel = settings.get().advanced.log_level;
+const validLevels = ['info', 'error', 'warn', 'debug'];
+assert(validLevels.includes(z2mLevel), `'${z2mLevel}' is not valid log_level, use one of '${validLevels.join(', ')}'`);
+const level = z2mToWinstonLevel(z2mLevel);
+
+const levelWithCompensatedLength: {[s: string]: string} = {
+    'info': 'info ',
+    'error': 'error',
+    'warn': 'warn ',
+    'debug': 'debug',
+};
+
+const timestampFormat = (): string => moment().format(settings.get().advanced.timestamp_format);
+
+// Setup default console logger
+const transportsToUse: winston.transport[] = [
+    new winston.transports.Console({
+        level,
+        silent: !output.includes('console'),
+        format: winston.format.combine(
+            winston.format.timestamp({format: timestampFormat}),
+            winston.format.printf(/* istanbul ignore next */(info) => {
+                const {timestamp, level, message} = info;
+                const l = winstonToZ2mLevel(level as WinstonLogLevel);
+                const prefix = colorizer.colorize(l, `Zigbee2MQTT:${levelWithCompensatedLength[l]}`);
+                return `${prefix} ${timestamp.split('.')[0]}: ${message}`;
+            }),
+        ),
+    }),
+];
+
+// Add file logger when enabled
+// NOTE: the initiation of the logger, even when not added as transport tries to create the logging directory
+const transportFileOptions: KeyValue = {
+    filename: path.join(directory, logFilename),
+    json: false,
+    level,
+    format: winston.format.combine(
+        winston.format.timestamp({format: timestampFormat}),
+        winston.format.printf(/* istanbul ignore next */(info) => {
+            const {timestamp, level, message} = info;
+            const l = winstonToZ2mLevel(level as WinstonLogLevel);
+            return `${levelWithCompensatedLength[l]} ${timestamp.split('.')[0]}: ${message}`;
+        }),
+    ),
+};
+
+if (settings.get().advanced.log_rotation) {
+    transportFileOptions.tailable = true;
+    transportFileOptions.maxFiles = 3; // Keep last 3 files
+    transportFileOptions.maxsize = 10000000; // 10MB
+}
+
+if (output.includes('file')) {
+    transportsToUse.push(new winston.transports.File(transportFileOptions));
+}
+
+/* istanbul ignore next */
+if (output.includes('syslog')) {
+    // eslint-disable-next-line
+    require('winston-syslog').Syslog;
+    const options: KeyValue = {
+        app_name: 'Zigbee2MQTT',
+        format: winston.format.printf(/* istanbul ignore next */(info) => {
+            return `${info.message}`;
+        }),
+        ...settings.get().advanced.log_syslog,
+    };
+    if (options.hasOwnProperty('type')) options.type = options.type.toString();
+    // @ts-ignore
+    transportsToUse.push(new winston.transports.Syslog(options));
+}
+
+// Create logger
+const logger = winston.createLogger({transports: transportsToUse, levels: winston.config.syslog.levels});
+
+// Cleanup any old log directory.
+function cleanup(): void {
+    if (settings.get().advanced.log_directory.includes('%TIMESTAMP%')) {
+        const rootDirectory = path.join(directory, '..');
+
+        let directories = fs.readdirSync(rootDirectory).map((d) => {
+            d = path.join(rootDirectory, d);
+            return {path: d, birth: fs.statSync(d).mtime};
+        });
+
+        directories.sort((a: KeyValue, b: KeyValue) => b.birth - a.birth);
+        directories = directories.slice(10, directories.length);
+        directories.forEach((dir) => {
+            logger.debug(`Removing old log directory '${dir.path}'`);
+            rimraf.sync(dir.path);
+        });
+    }
+}
+
+// Print to user what logging is enabled
+function logOutput(): void {
+    if (output.includes('file')) {
+        if (output.includes('console')) {
+            logger.info(`Logging to console and directory: '${directory}' filename: ${logFilename}`);
+        } else {
+            logger.info(`Logging to directory: '${directory}' filename: ${logFilename}`);
+        }
+        cleanup();
+    } else if (output.includes('console')) {
+        logger.info(`Logging to console only'`);
+    }
+}
+
+function addTransport(transport: winston.transport): void {
+    transport.level = transportsToUse[0].level;
+    logger.add(transport);
+}
+
+function getLevel(): Z2MLogLevel {
+    return winstonToZ2mLevel(transportsToUse[0].level as WinstonLogLevel);
+}
+
+function setLevel(level: Z2MLogLevel): void {
+    logger.transports.forEach((transport) => transport.level = z2mToWinstonLevel(level as Z2MLogLevel));
+}
+
+function warn(message: string): void {
+    // winston.config.syslog.levels doesnt have warn, but is required for syslog.
+    logger.warning(message);
+}
+
+function warning(message: string): void {
+    logger.warning(message);
+}
+
+function info(message: string): void {
+    logger.info(message);
+}
+
+function debug(message: string): void {
+    logger.debug(message);
+}
+
+function error(message: string): void {
+    logger.error(message);
+}
+
+export default {
+    logOutput, warn, warning, error, info, debug, setLevel, getLevel, cleanup, addTransport, winston: logger,
+};

lib/util/settings.schema.json

@@ -0,0 +1,770 @@
+{
+  "type": "object",
+  "properties": {
+    "device_options": {
+      "type": "object"
+    },
+    "homeassistant": {
+      "title": "Home Assistant integration",
+      "type": "boolean",
+      "description": "Home Assistant integration (MQTT discovery)",
+      "default": false
+    },
+    "permit_join": {
+      "type": "boolean",
+      "default": false,
+      "title": "Permit join",
+      "description": "Allow new devices to join (re-applied at restart)"
+    },
+    "external_converters": {
+      "type": "array",
+      "title": "External converters",
+      "description": "You can define external converters to e.g. add support for a DiY device",
+      "requiresRestart": true,
+      "items": {
+        "type": "string"
+      },
+      "examples": ["DIYRuZ_FreePad.js"]
+    },
+    "availability": {
+      "oneOf": [
+        {
+          "type": "boolean",
+          "title": "Availability (boolean)"
+        },
+        {
+          "type": "object",
+          "title": "Availability (object)",
+          "properties": {
+            "active": {
+              "type": "object",
+              "title": "Active",
+              "requiresRestart": true,
+              "description": "Options for active devices (routers/mains powered)",
+              "properties": {
+                "timeout": {
+                  "type": "number",
+                  "title": "Timeout",
+                  "requiresRestart": true,
+                  "default": 10,
+                  "description": "Time after which an active device will be marked as offline in minutes"
+                }
+              }
+            },
+            "passive": {
+              "type": "object",
+              "title": "Active",
+              "requiresRestart": true,
+              "description": "Options for passive devices (routers/mains powered)",
+              "properties": {
+                "timeout": {
+                  "type": "number",
+                  "title": "Timeout",
+                  "requiresRestart": true,
+                  "default": 1500,
+                  "description": "Time after which an passive device will be marked as offline in minutes"
+                }
+              }
+            }
+          }
+        }
+      ],
+      "title": "Availability",
+      "requiresRestart": true,
+      "description": "Checks wether devices are online/offline"
+    },
+    "mqtt": {
+      "type": "object",
+      "title": "MQTT",
+      "properties": {
+        "base_topic": {
+          "type": "string",
+          "title": "Base topic",
+          "requiresRestart": true,
+          "description": "MQTT base topic for Zigbee2MQTT MQTT messages",
+          "examples": ["zigbee2mqtt"]
+        },
+        "server": {
+          "type": "string",
+          "title": "MQTT server",
+          "requiresRestart": true,
+          "description": "MQTT server URL (use mqtts:// for SSL/TLS connection)",
+          "examples": ["mqtt://localhost:1883"]
+        },
+        "keepalive": {
+          "type": "number",
+          "title": "Keepalive",
+          "requiresRestart": true,
+          "description": "MQTT keepalive in second",
+          "default": 60
+        },
+        "ca": {
+          "type": "string",
+          "title": "Certificate authority",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS certificate of CA used to sign server and client certificates",
+          "examples": ["/etc/ssl/mqtt-ca.crt"]
+        },
+        "key": {
+          "type": "string",
+          "title": "SSL/TLS key",
+          "requiresRestart": true,
+          "description": "Absolute path to SSL/TLS key for client-authentication",
+          "examples": ["/etc/ssl/mqtt-client.key"]
+        },
+        "cert": {
+          "type": "string",
+          "title": "SSL/TLS certificate",
+          "description": "Absolute path to SSL/TLS certificate for client-authentication",
+          "requiresRestart": true,
+          "examples": ["/etc/ssl/mqtt-client.crt"]
+        },
+        "user": {
+          "type": "string",
+          "title": "User",
+          "requiresRestart": true,
+          "description": "MQTT server authentication user",
+          "examples": ["johnnysilverhand"]
+        },
+        "password": {
+          "type": "string",
+          "title": "Password",
+          "requiresRestart": true,
+          "description": "MQTT server authentication password",
+          "examples": ["ILOVEPELMENI"]
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "requiresRestart": true,
+          "description": "MQTT client ID",
+          "examples": ["MY_CLIENT_ID"]
+        },
+        "reject_unauthorized": {
+          "type": "boolean",
+          "title": "Reject unauthorized",
+          "requiresRestart": true,
+          "description": "Disable self-signed SSL certificate",
+          "default": true
+        },
+        "include_device_information": {
+          "type": "boolean",
+          "title": "Include device information",
+          "description": "Include device information to mqtt messages",
+          "default": false
+        },
+        "version": {
+          "type": ["number", "null"],
+          "title": "Version",
+          "requiresRestart": true,
+          "description": "MQTT protocol version",
+          "default": 4,
+          "examples": [5]
+        },
+        "force_disable_retain": {
+          "type": "boolean",
+          "title": "Force disable retain",
+          "requiresRestart": true,
+          "description": "Disable retain for all send messages. ONLY enable if you MQTT broker doesn't support retained message (e.g. AWS IoT core, Azure IoT Hub, Google Cloud IoT core, IBM Watson IoT Platform). Enabling will break the Home Assistant integration",
+          "default": false
+        }
+      },
+      "required": ["base_topic", "server"]
+    },
+    "serial": {
+      "type": "object",
+      "title": "Serial",
+      "properties": {
+        "port": {
+          "type": ["string", "null"],
+          "title": "Port",
+          "requiresRestart": true,
+          "description": "Location of the adapter. To autodetect the port, set null",
+          "examples": ["/dev/ttyACM0"]
+        },
+        "disable_led": {
+          "type": "boolean",
+          "title": "Disable led",
+          "requiresRestart": true,
+          "description": "Disable LED of the adapter if supported",
+          "default": false
+        },
+        "adapter": {
+          "type": ["string"],
+          "enum": ["deconz", "zstack", "zigate", "ezsp", "auto"],
+          "title": "Adapter",
+          "default": "auto",
+          "requiresRestart": true,
+          "description": "Adapter type, not needed unless you are experiencing problems"
+        }
+      }
+    },
+    "blocklist": {
+      "title": "Blocklist",
+      "requiresRestart": true,
+      "description": "Block devices from the network (by ieeeAddr)",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "passlist": {
+      "title": "Passlist",
+      "requiresRestart": true,
+      "description": "Allow only certain devices to join the network (by ieeeAddr). Note that all devices not on the passlist will be removed from the network!",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "whitelist": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Whitelist (deprecated, use passlist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "ban": {
+      "readOnly": true,
+      "type": "array",
+      "requiresRestart": true,
+      "title": "Ban (deprecated, use blocklist)",
+      "items": {
+        "type": "string"
+      }
+    },
+    "experimental": {
+      "type": "object",
+      "title": "Experimental",
+      "properties": {
+        "transmit_power": {
+          "type": ["number", "null"],
+          "title": "Transmit power",
+          "requiresRestart": true,
+          "description": "Transmit power of adapter, only available for Z-Stack (CC253*/CC2652/CC1352) adapters, CC2652 = 5dbm, CC1352 max is = 20dbm (5dbm default)"
+        },
+        "output": {
+          "type": "string",
+          "enum": ["attribute_and_json", "attribute", "json"],
+          "title": "MQTT output type",
+          "description": "Examples when 'state' of a device is published json: topic: 'zigbee2mqtt/my_bulb' payload '{\"state\": \"ON\"}' attribute: topic 'zigbee2mqtt/my_bulb/state' payload 'ON' attribute_and_json: both json and attribute (see above)"
+        }
+      }
+    },
+    "advanced": {
+      "type": "object",
+      "title": "Advanced",
+      "properties": {
+        "legacy_api": {
+          "type": "boolean",
+          "title": "Legacy API",
+          "requiresRestart": true,
+          "description": "Disables the legacy api (false = disable)",
+          "default": true
+        },
+        "pan_id": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Pan ID (string)"
+            },
+            {
+              "type": "number",
+              "title": "Pan ID (number)"
+            }
+          ],
+          "title": "Pan ID",
+          "requiresRestart": true,
+          "description": "ZigBee pan ID, changing requires repairing all devices!"
+        },
+        "ext_pan_id": {
+          "type": "array",
+          "items": {
+            "type": "number"
+          },
+          "title": "Ext Pan ID",
+          "requiresRestart": true,
+          "description": "Zigbee extended pan ID, changing requires repairing all devices!"
+        },
+        "channel": {
+          "type": "number",
+          "minimum": 11,
+          "maximum": 26,
+          "default": 11,
+          "title": "ZigBee channel",
+          "requiresRestart": true,
+          "description": "Zigbee channel, changing requires repairing all devices! (Note: use a ZLL channel: 11, 15, 20, or 25 to avoid Problems)",
+          "examples": [15, 20, 25]
+        },
+        "cache_state": {
+          "type": "boolean",
+          "title": "Cache state",
+          "description": "MQTT message payload will contain all attributes, not only changed ones. Has to be true when integrating via Home Assistant",
+          "default": true
+        },
+        "cache_state_persistent": {
+          "type": "boolean",
+          "title": "Persist cache state",
+          "description": "Persist cached state, only used when cache_state: true",
+          "default": true
+        },
+        "cache_state_send_on_startup": {
+          "type": "boolean",
+          "title": "Send cached state on startup",
+          "description": "Send cached state on startup, only used when cache_state: true",
+          "default": true
+        },
+        "log_rotation": {
+          "type": "boolean",
+          "title": "Log rotation",
+          "requiresRestart": true,
+          "description": "Log rotation",
+          "default": true
+        },
+        "log_symlink_current": {
+          "type": "boolean",
+          "title": "Log symlink current",
+          "requiresRestart": true,
+          "description": "Create symlink to current logs in the log directory",
+          "default": false
+        },
+        "log_level": {
+          "type": "string",
+          "enum": ["info", "warn", "error", "debug"],
+          "title": "Log level",
+          "description": "Logging level",
+          "default": "info"
+        },
+        "log_output": {
+          "type": "array",
+          "requiresRestart": true,
+          "items": {
+            "type": "string",
+            "enum": ["console", "file", "syslog"]
+          },
+          "title": "Log output",
+          "description": "Output location of the log, leave empty to supress logging"
+        },
+        "log_directory": {
+          "type": "string",
+          "title": "Log directory",
+          "requiresRestart": true,
+          "description": "Location of log directory",
+          "examples": ["data/log/%TIMESTAMP%"]
+        },
+        "log_file": {
+          "type": "string",
+          "title": "Log file",
+          "requiresRestart": true,
+          "description": "Log file name, can also contain timestamp",
+          "examples": ["zigbee2mqtt_%TIMESTAMP%.log"],
+          "default": "log.txt"
+        },
+        "baudrate": {
+          "type": "number",
+          "title": "Baudrate",
+          "requiresRestart": true,
+          "description": "Baud rate speed for serial port, this can be anything firmware support but default is 115200 for Z-Stack and EZSP, 38400 for Deconz, however note that some EZSP firmware need 57600",
+          "examples": [38400, 57600, 115200]
+        },
+        "rtscts": {
+          "type": "boolean",
+          "title": "RTS / CTS",
+          "requiresRestart": true,
+          "description": "RTS / CTS Hardware Flow Control for serial port"
+        },
+        "soft_reset_timeout": {
+          "type": "number",
+          "minimum": 0,
+          "requiresRestart": true,
+          "title": "Soft reset timeout (deprecated)",
+          "description": "Soft reset ZNP after timeout",
+          "readOnly": true
+        },
+        "network_key": {
+          "oneOf": [{
+              "type": "string",
+              "title": "Network key(string)"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "number"
+              },
+              "title": "Network key(array)"
+            }
+          ],
+          "title": "Network key",
+          "requiresRestart": true,
+          "description": "Network encryption key, changing requires repairing all devices!"
+        },
+        "last_seen": {
+          "type": "string",
+          "enum": ["disable", "ISO_8601", "ISO_8601_local", "epoch"],
+          "title": "Last seen",
+          "description": "Add a last_seen attribute to MQTT messages, contains date/time of last Zigbee message",
+          "default": "disable"
+        },
+        "elapsed": {
+          "type": "boolean",
+          "title": "Elapsed",
+          "description": "Add an elapsed attribute to MQTT messages, contains milliseconds since the previous msg",
+          "default": false
+        },
+        "report": {
+          "type": "boolean",
+          "title": "Reporting",
+          "requiresRestart": true,
+          "readOnly": true,
+          "description": "Enables report feature (deprecated)"
+        },
+        "homeassistant_discovery_topic": {
+          "type": "string",
+          "title": "Homeassistant discovery topic",
+          "description": "Home Assistant discovery topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant"]
+        },
+        "homeassistant_legacy_entity_attributes": {
+          "type": "boolean",
+          "title": "Home Assistant legacy entity attributes",
+          "description": "Home Assistant legacy entity attributes, when enabled Zigbee2MQTT will add state attributes to each entity, additional to the separate entities and devices it already creates",
+          "default": true
+        },
+        "homeassistant_status_topic": {
+          "type": "string",
+          "title": "Home Assistant status topic",
+          "description": "Home Assistant status topic",
+          "requiresRestart": true,
+          "examples": ["homeassistant/status"]
+        },
+        "timestamp_format": {
+          "type": "string",
+          "title": "Timestamp format",
+          "requiresRestart": true,
+          "description": "Log timestamp format",
+          "examples": ["YYYY-MM-DD HH:mm:ss"]
+        },
+        "adapter_concurrent": {
+          "title": "Adapter concurrency",
+          "requiresRestart": true,
+          "type": ["number", "null"],
+          "description": "Adapter concurrency (e.g. 2 for CC2531 or 16 for CC26X2R1) (default: null, uses recommended value)"
+        },
+        "adapter_delay": {
+          "type": ["number", "null"],
+          "requiresRestart": true,
+          "title": "Adapter delay",
+          "description": "Adapter delay"
+        },
+        "ikea_ota_use_test_url": {
+          "type": "boolean",
+          "title": "IKEA TRADFRI OTA use test url",
+          "requiresRestart": true,
+          "description": "Use IKEA TRADFRI OTA test server, see OTA updates documentation",
+          "default": false
+        },
+        "homeassistant_legacy_triggers": {
+          "type": "boolean",
+          "title": "Home Assistant legacy triggers",
+          "description": "Home Assistant legacy triggers, when enabled Zigbee2mqt will send an empty 'action' or 'click' after one has been send. A 'sensor_action' and 'sensor_click' will be discoverd",
+          "default": true
+        },
+        "log_syslog": {
+          "type": "object",
+          "title": "syslog",
+          "properties": {
+            "host": {
+              "type": "string",
+              "title": "Host",
+              "description": "The host running syslogd, defaults to localhost.",
+              "default": "localhost"
+            },
+            "port": {
+              "type": "number",
+              "title": "Port",
+              "description": "The port on the host that syslog is running on, defaults to syslogd's default port.",
+              "default": 123
+            },
+            "protocol": {
+              "type": "string",
+              "title": "Protocol",
+              "description": "The network protocol to log over (e.g. tcp4, udp4, tls4, unix, unix-connect, etc).",
+              "default": "tcp4",
+              "examples": [
+                "udp4",
+                "tls4",
+                "unix",
+                "unix-connect"
+              ]
+            },
+            "path": {
+              "type": "string",
+              "title": "Path",
+              "description": "The path to the syslog dgram socket (i.e. /dev/log or /var/run/syslog for OS X).",
+              "default": "/dev/log",
+              "examples": [
+                "/var/run/syslog"
+              ]
+            },
+            "pid": {
+              "type": "string",
+              "title": "PID",
+              "description": "PID of the process that log messages are coming from (Default process.pid).",
+              "default": "process.pid"
+            },
+            "localhost": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "Host to indicate that log messages are coming from (Default: localhost).",
+              "default": "localhost"
+            },
+            "type": {
+              "type": "string",
+              "title": "Type",
+              "description": "The type of the syslog protocol to use (Default: BSD, also valid: 5424).",
+              "default": "5424"
+            },
+            "app_name": {
+              "type": "string",
+              "title": "Localhost",
+              "description": "The name of the application (Default: Zigbee2MQTT).",
+              "default": "Zigbee2MQTT"
+            },
+            "eol": {
+              "type": "string",
+              "title": "eol",
+              "description": "The end of line character to be added to the end of the message (Default: Message without modifications).",
+              "default": "/n"
+            }
+          }
+        }
+      }
+    },
+    "map_options": {
+      "type": "object",
+      "title": "Networkmap",
+      "properties": {
+        "graphviz": {
+          "type": "object",
+          "properties": {
+            "colors": {
+              "type": "object",
+              "properties": {
+                "fill": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "font": {
+                  "type": "object",
+                  "properties": {
+                    "enddevice": {
+                      "type": "string"
+                    },
+                    "coordinator": {
+                      "type": "string"
+                    },
+                    "router": {
+                      "type": "string"
+                    }
+                  }
+                },
+                "line": {
+                  "type": "object",
+                  "properties": {
+                    "active": {
+                      "type": "string"
+                    },
+                    "inactive": {
+                      "type": "string"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "ota": {
+      "type": "object",
+      "title": "OTA updates",
+      "properties": {
+        "update_check_interval": {
+          "type": "number",
+          "title": "Update check interval",
+          "description": "Your device may request a check for a new firmware update. This value determines how frequently third party servers may actually be contacted to look for firmware updates. The value is set in minutes, and the default is 1 day.",
+          "default": 1440
+        },
+        "disable_automatic_update_check": {
+          "type": "boolean",
+          "title": "Disable automatic update check",
+          "description": "Zigbee devices may request a firmware update, and do so frequently, causing Zigbee2MQTT to reach out to third party servers. If you disable these device initiated checks, you can still initiate a firmware update check manually.",
+          "default": false
+        }
+      }
+    },
+    "devices": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^0x[\\d\\w]{16}$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/device"
+        }
+      }
+    },
+    "groups": {
+      "type": "object",
+      "propertyNames": {
+        "pattern": "^[\\w].*$"
+      },
+      "patternProperties": {
+        "^.*$": {
+          "$ref": "#/definitions/group"
+        }
+      }
+    },
+    "frontend": {
+      "type": "object",
+      "title": "Frontend",
+      "properties": {
+        "port": {
+          "type": "number",
+          "title": "Port",
+          "description": "Frontend binding port",
+          "default": 8080,
+          "requiresRestart": true
+        },
+        "host": {
+          "type": "string",
+          "title": "Bind host",
+          "description": "Frontend binding host",
+          "default":" 0.0.0.0",
+          "requiresRestart": true
+        },
+        "auth_token": {
+          "type": ["string", "null"],
+          "title": "Auth token",
+          "description": "Enables authentication, disabled by default",
+          "requiresRestart": true
+        }
+      }
+    }
+  },
+  "required": ["mqtt"],
+  "definitions": {
+    "device": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string",
+          "title": "Friendly name",
+          "description": "Used in the MQTT topic of a device. By default this is the device ID",
+          "readOnly": true
+        },
+        "retain": {
+          "type": "boolean",
+          "title": "Retain",
+          "description": "Retain MQTT messages of this device"
+        },
+        "retention": {
+          "type": "number",
+          "title": "Retention",
+          "description": "Sets the MQTT Message Expiry in seconds, Make sure to set mqtt.version to 5"
+        },
+        "qos": {
+          "type": "number",
+          "title": "QoS",
+          "description": "QoS level for MQTT messages of this device"
+        },
+        "debounce": {
+          "type": "number",
+          "title": "Debounce",
+          "description": "Debounces messages of this device"
+        },
+        "debounce_ignore": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["action"],
+          "title": "Ignore debounce",
+          "description": "Protects unique payload values of specified payload properties from overriding within debounce time"
+        },
+        "optimistic": {
+          "type": "boolean",
+          "title": "Optimistic",
+          "description": "Publish optimistic state after set",
+          "default": true
+        },
+        "filtered_optimistic": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["color_mode", "color_temp", "color"],
+          "title": "Filtered optimistic attributes",
+          "description": "Filter attributes from optimistic publish payload when calling /set. (This has no effect if optimistic is set to false)."
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "examples": ["temperature", "battery", "action"],
+          "title": "Filtered publish attributes",
+          "description": "Filter attributes from publish payload."
+        },
+        "icon": {
+          "type": "string",
+          "title": "Icon",
+          "description": "The user-defined device icon for the frontend. It can be a link to an image (not a path to a file) or base64 encoded data URL like: image/svg+xml;base64,PHN2ZyB3aW....R0aD"
+        }
+      },
+      "required": ["friendly_name"]
+    },
+    "group": {
+      "type": "object",
+      "properties": {
+        "friendly_name": {
+          "type": "string"
+        },
+        "retain": {
+          "type": "boolean"
+        },
+        "devices": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "optimistic": {
+          "type": "boolean"
+        },
+        "qos": {
+          "type": "number"
+        },
+        "filtered_attributes": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "required": ["friendly_name"]
+    }
+  }
+}

lib/util/settings.ts

@@ -1,13 +1,19 @@
-const data = require('./data');
-const utils = require('./utils');
-const file = process.env.ZIGBEE2MQTT_CONFIG || data.joinPath('configuration.yaml');
-const objectAssignDeep = require(`object-assign-deep`);
-const path = require('path');
-const yaml = require('./yaml');
-const Ajv = require('ajv');
-const ajv = new Ajv({allErrors: true});
+import data from './data';
+import utils from './utils';
+import objectAssignDeep from 'object-assign-deep';
+import path from 'path';
+import yaml from './yaml';
+import Ajv from 'ajv';
+import schemaJson from './settings.schema.json';
+export const schema = schemaJson;
 
-const defaults = {
+// DEPRECATED ZIGBEE2MQTT_CONFIG: https://github.com/Koenkk/zigbee2mqtt/issues/4697
+const file = process.env.ZIGBEE2MQTT_CONFIG ?? data.joinPath('configuration.yaml');
+const ajvSetting = new Ajv({allErrors: true}).addKeyword('requiresRestart').compile(schema);
+const ajvRestartRequired = new Ajv({allErrors: true})
+    .addKeyword({keyword: 'requiresRestart', validate: (schema: unknown) => !schema}).compile(schema);
+
+const defaults: RecursivePartial<Settings> = {
     passlist: [],
     blocklist: [],
     // Deprecated: use block/passlist
@@ -16,6 +22,11 @@ const defaults = {
     permit_join: false,
     mqtt: {
         include_device_information: false,
+        /**
+         * Configurable force disable retain flag on mqtt publish.
+         * https://github.com/Koenkk/zigbee2mqtt/pull/4948
+         */
+        force_disable_retain: false,
     },
     serial: {
         disable_led: false,
@@ -44,11 +55,11 @@ const defaults = {
     experimental: {
         // json or attribute or attribute_and_json
         output: 'json',
-        new_api: false,
     },
     advanced: {
         legacy_api: true,
         log_rotation: true,
+        log_symlink_current: false,
         log_output: ['console', 'file'],
         log_directory: path.join(data.getPath(), 'log', '%TIMESTAMP%'),
         log_file: 'log.txt',
@@ -59,9 +70,9 @@ const defaults = {
         ext_pan_id: [0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD, 0xDD],
         channel: 11,
         adapter_concurrent: null,
+        adapter_delay: null,
 
         // Availability timeout in seconds, disabled by default.
-        availability_timeout: 0,
         availability_blocklist: [],
         availability_passlist: [],
         // Deprecated, use block/passlist
@@ -115,6 +126,15 @@ const defaults = {
          */
         homeassistant_status_topic: 'hass/status',
 
+        /**
+         * Home Assistant legacy entity attributes, when enabled:
+         * Zigbee2MQTT will send additional states as attributes with each entity.
+         * For example, A temperature & humidity sensor will have 2 entities for
+         * the temperature and humidity, with this setting enabled both entities
+         * will also have an temperature and humidity attribute.
+         */
+        homeassistant_legacy_entity_attributes: true,
+
         /**
          * Home Assistant legacy triggers, when enabled:
          * - Zigbee2mqt will send an empty 'action' or 'click' after one has been send
@@ -128,185 +148,37 @@ const defaults = {
          */
         timestamp_format: 'YYYY-MM-DD HH:mm:ss',
     },
+    ota: {
+        /**
+         * Minimal time delta in minutes between polling third party server for potential firmware updates
+         */
+        update_check_interval: 24 * 60,
+        /**
+         * Completely disallow Zigbee devices to initiate a search for a potential firmware update.
+         * If set to true, only a user-initiated update search will be possible.
+         */
+        disable_automatic_update_check: false,
+    },
     external_converters: [],
 };
 
-const schema = {
-    type: 'object',
-    properties: {
-        device_options: {type: 'object'},
-        homeassistant: {type: 'boolean'},
-        permit_join: {type: 'boolean'},
-        mqtt: {
-            type: 'object',
-            properties: {
-                base_topic: {type: 'string'},
-                server: {type: 'string'},
-                keepalive: {type: 'number'},
-                ca: {type: 'string'},
-                key: {type: 'string'},
-                cert: {type: 'string'},
-                user: {type: 'string'},
-                password: {type: 'string'},
-                client_id: {type: 'string'},
-                reject_unauthorized: {type: 'boolean'},
-                include_device_information: {type: 'boolean'},
-                version: {type: 'number'},
-            },
-            required: ['base_topic', 'server'],
-        },
-        serial: {
-            type: 'object',
-            properties: {
-                port: {type: ['string', 'null']},
-                disable_led: {type: 'boolean'},
-                adapter: {type: 'string', enum: ['deconz', 'zstack']},
-            },
-        },
-        blocklist: {type: 'array', items: {type: 'string'}},
-        passlist: {type: 'array', items: {type: 'string'}},
-        // Deprecated: use block/passlist
-        whitelist: {type: 'array', items: {type: 'string'}},
-        ban: {type: 'array', items: {type: 'string'}},
-        experimental: {
-            type: 'object',
-            properties: {
-                transmit_power: {type: 'number'},
-                output: {type: 'string', enum: ['attribute_and_json', 'attribute', 'json']},
-            },
-        },
-        advanced: {
-            type: 'object',
-            properties: {
-                legacy_api: {type: 'boolean'},
-                pan_id: {type: 'number'},
-                ext_pan_id: {type: 'array', items: {type: 'number'}},
-                channel: {type: 'number', minimum: 11, maximum: 26},
-                cache_state: {type: 'boolean'},
-                cache_state_persistent: {type: 'boolean'},
-                cache_state_send_on_startup: {type: 'boolean'},
-                log_rotation: {type: 'boolean'},
-                log_level: {type: 'string', enum: ['info', 'warn', 'error', 'debug']},
-                log_output: {type: 'array', items: {type: 'string'}},
-                log_directory: {type: 'string'},
-                log_file: {type: 'string'},
-                baudrate: {type: 'number'},
-                rtscts: {type: 'boolean'},
-                soft_reset_timeout: {type: 'number', minimum: 0},
-                network_key: {type: ['array', 'string'], items: {type: 'number'}},
-                last_seen: {type: 'string', enum: ['disable', 'ISO_8601', 'ISO_8601_local', 'epoch']},
-                elapsed: {type: 'boolean'},
-                availability_timeout: {type: 'number', minimum: 0},
-                availability_blocklist: {type: 'array', items: {type: 'string'}},
-                availability_passlist: {type: 'array', items: {type: 'string'}},
-                // Deprecated, use block/passlist
-                availability_blacklist: {type: 'array', items: {type: 'string'}},
-                availability_whitelist: {type: 'array', items: {type: 'string'}},
-                report: {type: 'boolean'},
-                homeassistant_discovery_topic: {type: 'string'},
-                homeassistant_status_topic: {type: 'string'},
-                timestamp_format: {type: 'string'},
-                adapter_concurrent: {type: 'number'},
-            },
-        },
-        map_options: {
-            type: 'object',
-            properties: {
-                graphviz: {
-                    type: 'object',
-                    properties: {
-                        colors: {
-                            type: 'object',
-                            properties: {
-                                fill: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                font: {
-                                    type: 'object',
-                                    properties: {
-                                        enddevice: {type: 'string'},
-                                        coordinator: {type: 'string'},
-                                        router: {type: 'string'},
-                                    },
-                                },
-                                line: {
-                                    type: 'object',
-                                    properties: {
-                                        active: {type: 'string'},
-                                        inactive: {type: 'string'},
-                                    },
-                                },
-                            },
-                        },
-                    },
-                },
-            },
-        },
-        devices: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^0x[\\d\\w]{16}$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        retention: {type: 'number'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-        groups: {
-            type: 'object',
-            propertyNames: {
-                pattern: '^[\\w].*$',
-            },
-            patternProperties: {
-                '^.*$': {
-                    type: 'object',
-                    properties: {
-                        friendly_name: {type: 'string'},
-                        retain: {type: 'boolean'},
-                        devices: {type: 'array', items: {type: 'string'}},
-                        optimistic: {type: 'boolean'},
-                        qos: {type: 'number'},
-                        filtered_attributes: {type: 'array', items: {type: 'string'}},
-                    },
-                    required: ['friendly_name'],
-                },
-            },
-        },
-    },
-    external_converters: {
-        type: 'array',
-        items: {type: 'string'},
-    },
-    required: ['homeassistant', 'permit_join', 'mqtt'],
-};
+let _settings: Partial<Settings>;
+let _settingsWithDefaults: Settings;
 
-
-let _settings;
-let _settingsWithDefaults;
-
-function write() {
-    const settings = get();
-    const toWrite = objectAssignDeep.noMutate(settings);
+function write(): void {
+    const settings = getInternalSettings();
+    const toWrite: KeyValue = objectAssignDeep({}, settings);
 
     // Read settings to check if we have to split devices/groups into separate file.
     const actual = yaml.read(file);
 
     // In case the setting is defined in a separte file (e.g. !secret network_key) update it there.
-    for (const path of [['mqtt', 'user'], ['mqtt', 'password'], ['advanced', 'network_key']]) {
+    for (const path of [
+        ['mqtt', 'user'],
+        ['mqtt', 'password'],
+        ['advanced', 'network_key'],
+        ['frontend', 'auth_token'],
+    ]) {
         if (actual[path[0]] && actual[path[0]][path[1]]) {
             const match = /!(.*) (.*)/g.exec(actual[path[0]][path[1]]);
             if (match) {
@@ -316,25 +188,37 @@ function write() {
         }
     }
 
-    if (typeof actual.devices === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.devices), settings.devices);
-        toWrite.devices = actual.devices;
-    }
+    // Write devices/groups to separate file if required.
+    const writeDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof actual[type] === 'string' || Array.isArray(actual[type])) {
+            const fileToWrite = Array.isArray(actual[type]) ? actual[type][0] : actual[type];
+            const content = objectAssignDeep({}, settings[type]);
 
-    if (typeof actual.groups === 'string') {
-        yaml.writeIfChanged(data.joinPath(actual.groups), settings.groups);
-        toWrite.groups = actual.groups;
-    }
+            // If an array, only write to first file and only devices which are not in the other files.
+            if (Array.isArray(actual[type])) {
+                actual[type].filter((f: string, i: number) => i !== 0)
+                    .map((f: string) => yaml.readIfExists(data.joinPath(f), {}))
+                    .map((c: KeyValue) => Object.keys(c))
+                    .forEach((k: string) => delete content[k]);
+            }
+
+            yaml.writeIfChanged(data.joinPath(fileToWrite), content);
+            toWrite[type] = actual[type];
+        }
+    };
+
+    writeDevicesOrGroups('devices');
+    writeDevicesOrGroups('groups');
 
     yaml.writeIfChanged(file, toWrite);
 
     _settings = read();
-    _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+    _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
 }
 
-function validate() {
+export function validate(): string[] {
     try {
-        get();
+        getInternalSettings();
     } catch (error) {
         if (error.name === 'YAMLException') {
             return [
@@ -346,10 +230,8 @@ function validate() {
         return [error.message];
     }
 
-    const validate = ajv.compile(schema);
-    validate(_settings);
-    if (validate.errors) {
-        return validate.errors.map((v) => `${v.dataPath.substring(1)} ${v.message}`);
+    if (!ajvSetting(_settings)) {
+        return ajvSetting.errors.map((v) => `${v.instancePath.substring(1)} ${v.message}`);
     }
 
     const errors = [];
@@ -358,15 +240,20 @@ function validate() {
         errors.push(`advanced.network_key: should be array or 'GENERATE' (is '${_settings.advanced.network_key}')`);
     }
 
+    if (_settings.advanced && _settings.advanced.pan_id && typeof _settings.advanced.pan_id === 'string' &&
+        _settings.advanced.pan_id !== 'GENERATE') {
+        errors.push(`advanced.pan_id: should be number or 'GENERATE' (is '${_settings.advanced.pan_id}')`);
+    }
+
     // Verify that all friendly names are unique
-    const names = [];
-    const check = (name) => {
+    const names: string[] = [];
+    const check = (name: string): void => {
         if (names.includes(name)) errors.push(`Duplicate friendly_name '${name}' found`);
         errors.push(...utils.validateFriendlyName(name));
         names.push(name);
     };
 
-    const settingsWithDefaults = getWithDefaults();
+    const settingsWithDefaults = get();
     Object.values(settingsWithDefaults.devices).forEach((d) => check(d.friendly_name));
     Object.values(settingsWithDefaults.groups).forEach((g) => check(g.friendly_name));
 
@@ -378,9 +265,9 @@ function validate() {
         }
     }
 
-    const checkAvailabilityList = (list, type) => {
+    const checkAvailabilityList = (list: string[], type: string): void => {
         list.forEach((e) => {
-            if (!getEntity(e)) {
+            if (!getDevice(e)) {
                 errors.push(`Non-existing entity '${e}' specified in '${type}'`);
             }
         });
@@ -394,11 +281,12 @@ function validate() {
     return errors;
 }
 
-function read() {
-    const s = yaml.read(file);
+function read(): Settings {
+    const s = yaml.read(file) as Settings;
 
     // Read !secret MQTT username and password if set
-    const interpetValue = (value) => {
+    // eslint-disable-next-line
+    const interpetValue = (value: any): any => {
         const re = /!(.*) (.*)/g;
         const match = re.exec(value);
         if (match) {
@@ -410,59 +298,72 @@ function read() {
         }
     };
 
-    if (s.mqtt && s.mqtt.user && s.mqtt.password) {
+    if (s.mqtt?.user && s.mqtt?.password) {
         s.mqtt.user = interpetValue(s.mqtt.user);
         s.mqtt.password = interpetValue(s.mqtt.password);
     }
 
-    if (s.advanced && s.advanced.network_key) {
+    if (s.advanced?.network_key) {
         s.advanced.network_key = interpetValue(s.advanced.network_key);
     }
 
-    // Read devices/groups configuration from separate file.
-    if (typeof s.devices === 'string') {
-        const file = data.joinPath(s.devices);
-        s.devices = yaml.readIfExists(file) || {};
+    if (s.frontend?.auth_token) {
+        s.frontend.auth_token = interpetValue(s.frontend.auth_token);
     }
 
-    if (typeof s.groups === 'string') {
-        const file = data.joinPath(s.groups);
-        s.groups = yaml.readIfExists(file) || {};
-    }
+    // Read devices/groups configuration from separate file if specified.
+    const readDevicesOrGroups = (type: 'devices' | 'groups'): void => {
+        if (typeof s[type] === 'string' || Array.isArray(s[type])) {
+            /* eslint-disable-line */ // @ts-ignore
+            const files: string[] = Array.isArray(s[type]) ? s[type] : [s[type]];
+            s[type] = {};
+            for (const file of files) {
+                const content = yaml.readIfExists(data.joinPath(file), {});
+                /* eslint-disable-line */ // @ts-ignore
+                s[type] = objectAssignDeep.noMutate(s[type], content);
+            }
+        }
+    };
+
+    readDevicesOrGroups('devices');
+    readDevicesOrGroups('groups');
 
     return s;
 }
 
-function applyEnvironmentVariables(settings) {
-    const iterate = (obj, path) => {
+function applyEnvironmentVariables(settings: Partial<Settings>): void {
+    const iterate = (obj: KeyValue, path: string[]): void => {
         Object.keys(obj).forEach((key) => {
             if (key !== 'type') {
-                if (key !== 'properties') {
+                if (key !== 'properties' && obj[key]) {
                     const type = (obj[key].type || 'object').toString();
                     const envPart = path.reduce((acc, val) => `${acc}${val}_`, '');
                     const envVariableName = (`ZIGBEE2MQTT_CONFIG_${envPart}${key}`).toUpperCase();
                     if (process.env[envVariableName]) {
-                        const setting = path.reduce((acc, val, index) => {
+                        const setting = path.reduce((acc, val) => {
+                            /* eslint-disable-line */ // @ts-ignore
                             acc[val] = acc[val] || {};
+                            /* eslint-disable-line */ // @ts-ignore
                             return acc[val];
                         }, settings);
 
                         if (type.indexOf('object') >= 0 || type.indexOf('array') >= 0) {
                             setting[key] = JSON.parse(process.env[envVariableName]);
-                        }
-                        if (type.indexOf('number') >= 0) {
+                        } else if (type.indexOf('number') >= 0) {
+                            /* eslint-disable-line */ // @ts-ignore
                             setting[key] = process.env[envVariableName] * 1;
-                        }
-                        if (type.indexOf('boolean') >= 0) {
+                        } else if (type.indexOf('boolean') >= 0) {
                             setting[key] = process.env[envVariableName].toLowerCase() === 'true';
-                        }
-                        if (type.indexOf('string') >= 0) {
-                            setting[key] = process.env[envVariableName];
+                        } else {
+                            /* istanbul ignore else */
+                            if (type.indexOf('string') >= 0) {
+                                setting[key] = process.env[envVariableName];
+                            }
                         }
                     }
                 }
 
-                if (typeof obj[key] === 'object') {
+                if (typeof obj[key] === 'object' && obj[key]) {
                     const newPath = [...path];
                     if (key !== 'properties') {
                         newPath.push(key);
@@ -475,7 +376,7 @@ function applyEnvironmentVariables(settings) {
     iterate(schema.properties, []);
 }
 
-function get() {
+function getInternalSettings(): Partial<Settings> {
     if (!_settings) {
         _settings = read();
         applyEnvironmentVariables(_settings);
@@ -484,9 +385,9 @@ function get() {
     return _settings;
 }
 
-function getWithDefaults() {
+export function get(): Settings {
     if (!_settingsWithDefaults) {
-        _settingsWithDefaults = objectAssignDeep.noMutate(defaults, get());
+        _settingsWithDefaults = objectAssignDeep({}, defaults, getInternalSettings()) as Settings;
     }
 
     if (!_settingsWithDefaults.devices) {
@@ -500,8 +401,9 @@ function getWithDefaults() {
     return _settingsWithDefaults;
 }
 
-function set(path, value) {
-    let settings = get();
+export function set(path: string[], value: string | number | boolean | KeyValue): void {
+    /* eslint-disable-next-line */
+    let settings: any = getInternalSettings();
 
     for (let i = 0; i < path.length; i++) {
         const key = path[i];
@@ -519,30 +421,49 @@ function set(path, value) {
     write();
 }
 
-function getGroup(IDorName) {
-    const settings = getWithDefaults();
+export function apply(newSettings: Record<string, unknown>): boolean {
+    ajvSetting(newSettings);
+    const errors = ajvSetting.errors && ajvSetting.errors.filter((e) => e.keyword !== 'required');
+    if (errors.length) {
+        const error = errors[0];
+        throw new Error(`${error.instancePath.substring(1)} ${error.message}`);
+    }
+
+    getInternalSettings(); // Ensure _settings is intialized.
+    /* eslint-disable-line */ // @ts-ignore
+    _settings = objectAssignDeep.noMutate(_settings, newSettings);
+    write();
+
+    ajvRestartRequired(newSettings);
+    const restartRequired = ajvRestartRequired.errors &&
+        !!ajvRestartRequired.errors.find((e) => e.keyword === 'requiresRestart');
+    return restartRequired;
+}
+
+export function getGroup(IDorName: string | number): GroupSettings {
+    const settings = get();
     const byID = settings.groups[IDorName];
     if (byID) {
-        return {optimistic: true, devices: [], ...byID, ID: Number(IDorName), friendlyName: byID.friendly_name};
+        return {devices: [], ...byID, ID: Number(IDorName)};
     }
 
     for (const [ID, group] of Object.entries(settings.groups)) {
         if (group.friendly_name === IDorName) {
-            return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+            return {devices: [], ...group, ID: Number(ID)};
         }
     }
 
     return null;
 }
 
-function getGroups() {
-    const settings = getWithDefaults();
+export function getGroups(): GroupSettings[] {
+    const settings = get();
     return Object.entries(settings.groups).map(([ID, group]) => {
-        return {optimistic: true, devices: [], ...group, ID: Number(ID), friendlyName: group.friendly_name};
+        return {devices: [], ...group, ID: Number(ID)};
     });
 }
 
-function getGroupThrowIfNotExists(IDorName) {
+function getGroupThrowIfNotExists(IDorName: string): GroupSettings {
     const group = getGroup(IDorName);
     if (!group) {
         throw new Error(`Group '${IDorName}' does not exist`);
@@ -551,23 +472,23 @@ function getGroupThrowIfNotExists(IDorName) {
     return group;
 }
 
-function getDevice(IDorName) {
-    const settings = getWithDefaults();
+export function getDevice(IDorName: string): DeviceSettings {
+    const settings = get();
     const byID = settings.devices[IDorName];
     if (byID) {
-        return {...byID, ID: IDorName, friendlyName: byID.friendly_name};
+        return {...byID, ID: IDorName};
     }
 
     for (const [ID, device] of Object.entries(settings.devices)) {
         if (device.friendly_name === IDorName) {
-            return {...device, ID, friendlyName: device.friendly_name};
+            return {...device, ID};
         }
     }
 
     return null;
 }
 
-function getDeviceThrowIfNotExists(IDorName) {
+function getDeviceThrowIfNotExists(IDorName: string): DeviceSettings {
     const device = getDevice(IDorName);
     if (!device) {
         throw new Error(`Device '${IDorName}' does not exist`);
@@ -576,26 +497,12 @@ function getDeviceThrowIfNotExists(IDorName) {
     return device;
 }
 
-function getEntity(IDorName) {
-    const device = getDevice(IDorName);
-    if (device) {
-        return {...device, type: 'device'};
-    }
-
-    const group = getGroup(IDorName);
-    if (group) {
-        return {...group, type: 'group'};
-    }
-
-    return null;
-}
-
-function addDevice(ID) {
+export function addDevice(ID: string): DeviceSettings {
     if (getDevice(ID)) {
         throw new Error(`Device '${ID}' already exists`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
 
     if (!settings.devices) {
         settings.devices = {};
@@ -606,9 +513,8 @@ function addDevice(ID) {
     return getDevice(ID);
 }
 
-// Legacy: can be removed after bridgeLegacy has been removed
-function whitelistDevice(ID) {
-    const settings = get();
+export function whitelistDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.whitelist) {
         settings.whitelist = [];
     }
@@ -621,8 +527,8 @@ function whitelistDevice(ID) {
     write();
 }
 
-function blockDevice(ID) {
-    const settings = get();
+export function blockDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.blocklist) {
         settings.blocklist = [];
     }
@@ -631,8 +537,8 @@ function blockDevice(ID) {
     write();
 }
 
-function banDevice(ID) {
-    const settings = get();
+export function banDevice(ID: string): void {
+    const settings = getInternalSettings();
     if (!settings.ban) {
         settings.ban = [];
     }
@@ -641,19 +547,30 @@ function banDevice(ID) {
     write();
 }
 
-function removeDevice(IDorName) {
+export function removeDevice(IDorName: string): void {
     const device = getDeviceThrowIfNotExists(IDorName);
-    const settings = get();
+    const settings = getInternalSettings();
     delete settings.devices[device.ID];
+
+    // Remove device from groups
+    if (settings.groups) {
+        const regex =
+            new RegExp(`^(${device.friendly_name}|${device.ID})(/(\\d|${utils.endpointNames.join('|')}))?$`);
+        for (const group of Object.values(settings.groups).filter((g) => g.devices)) {
+            group.devices = group.devices.filter((device) => !device.match(regex));
+        }
+    }
+
     write();
 }
 
-function addGroup(name, ID=null) {
+export function addGroup(name: string, ID?: string): GroupSettings {
+    utils.validateFriendlyName(name, true);
     if (getGroup(name) || getDevice(name)) {
         throw new Error(`friendly_name '${name}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (!settings.groups) {
         settings.groups = {};
     }
@@ -668,7 +585,7 @@ function addGroup(name, ID=null) {
         // ensure provided ID is not in use
         ID = ID.toString();
         if (settings.groups.hasOwnProperty(ID)) {
-            throw new Error(`group id '${ID}' is already in use`);
+            throw new Error(`Group ID '${ID}' is already in use`);
         }
     }
 
@@ -678,62 +595,58 @@ function addGroup(name, ID=null) {
     return getGroup(ID);
 }
 
-function groupHasDevice(group, keys) {
-    for (const device of group.devices) {
-        const index = keys.indexOf(device);
-        if (index != -1) {
-            return keys[index];
-        }
+function groupGetDevice(group: {devices?: string[]}, keys: string[]): string {
+    for (const device of group.devices ?? []) {
+        if (keys.includes(device)) return device;
     }
 
-    return false;
+    return null;
 }
 
-function addDeviceToGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function addDeviceToGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
 
     const group = settings.groups[groupID];
-    if (!group.devices) {
-        group.devices = [];
-    }
-
-    if (!groupHasDevice(group, keys)) {
+    if (!groupGetDevice(group, keys)) {
+        if (!group.devices) group.devices = [];
         group.devices.push(keys[0]);
         write();
     }
 }
 
-function removeDeviceFromGroup(groupIDorName, keys) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeDeviceFromGroup(IDorName: string, keys: string[]): void {
+    const groupID = getGroupThrowIfNotExists(IDorName).ID;
+    const settings = getInternalSettings();
     const group = settings.groups[groupID];
     if (!group.devices) {
-        group.devices = [];
+        return;
     }
 
-    const key = groupHasDevice(group, keys);
+    const key = groupGetDevice(group, keys);
     if (key) {
         group.devices = group.devices.filter((d) => d != key);
         write();
     }
 }
 
-function removeGroup(groupIDorName) {
-    const groupID = getGroupThrowIfNotExists(groupIDorName).ID;
-    const settings = get();
+export function removeGroup(IDorName: string | number): void {
+    const groupID = getGroupThrowIfNotExists(IDorName.toString()).ID;
+    const settings = getInternalSettings();
     delete settings.groups[groupID];
     write();
 }
 
-function changeEntityOptions(IDorName, newOptions) {
-    const settings = get();
+export function changeEntityOptions(IDorName: string, newOptions: KeyValue): void {
+    const settings = getInternalSettings();
     delete newOptions.friendly_name;
     delete newOptions.devices;
     if (getDevice(IDorName)) {
         objectAssignDeep(settings.devices[getDevice(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.devices[getDevice(IDorName).ID]);
     } else if (getGroup(IDorName)) {
         objectAssignDeep(settings.groups[getGroup(IDorName).ID], newOptions);
+        utils.removeNullPropertiesFromObject(settings.groups[getGroup(IDorName).ID]);
     } else {
         throw new Error(`Device or group '${IDorName}' does not exist`);
     }
@@ -741,12 +654,13 @@ function changeEntityOptions(IDorName, newOptions) {
     write();
 }
 
-function changeFriendlyName(IDorName, newName) {
+export function changeFriendlyName(IDorName: string, newName: string): void {
+    utils.validateFriendlyName(newName, true);
     if (getGroup(newName) || getDevice(newName)) {
         throw new Error(`friendly_name '${newName}' is already in use`);
     }
 
-    const settings = get();
+    const settings = getInternalSettings();
     if (getDevice(IDorName)) {
         settings.devices[getDevice(IDorName).ID].friendly_name = newName;
     } else if (getGroup(IDorName)) {
@@ -758,37 +672,18 @@ function changeFriendlyName(IDorName, newName) {
     write();
 }
 
-module.exports = {
-    validate,
-    get: getWithDefaults,
-    set,
-    getDevice,
-    getGroup,
-    getGroups,
-    getEntity,
-    whitelistDevice,
-    banDevice,
-    blockDevice,
-    addDevice,
-    removeDevice,
-    addGroup,
-    removeGroup,
-    addDeviceToGroup,
-    removeDeviceFromGroup,
-    changeEntityOptions,
-    changeFriendlyName,
+export function reRead(): void {
+    _settings = null;
+    getInternalSettings();
+    _settingsWithDefaults = null;
+    get();
+}
 
-    // For tests only
-    _write: write,
-    _reRead: () => {
-        _settings = null;
-        get();
-        _settingsWithDefaults = null;
-        getWithDefaults();
-    },
-    _clear: () => {
+export const testing = {
+    write,
+    clear: (): void => {
         _settings = null;
         _settingsWithDefaults = null;
     },
-    _getDefaults: () => defaults,
+    defaults,
 };

lib/util/utils.js

@@ -1,202 +0,0 @@
-const equals = require('fast-deep-equal/es6');
-const humanizeDuration = require('humanize-duration');
-const data = require('./data');
-
-// Xiaomi uses 4151 and 4447 (lumi.plug) as manufacturer ID.
-const xiaomiManufacturerID = [4151, 4447];
-const ikeaTradfriManufacturerID = [4476];
-
-// construct a local ISO8601 string (instead of UTC-based)
-// Example:
-//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
-//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
-function toLocalISOString(dDate) {
-    const tzOffset = -dDate.getTimezoneOffset();
-    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
-    const pad = function(num) {
-        const norm = Math.floor(Math.abs(num));
-        return (norm < 10 ? '0' : '') + norm;
-    };
-
-    return dDate.getFullYear() +
-        '-' + pad(dDate.getMonth() + 1) +
-        '-' + pad(dDate.getDate()) +
-        'T' + pad(dDate.getHours()) +
-        ':' + pad(dDate.getMinutes()) +
-        ':' + pad(dDate.getSeconds()) +
-        plusOrMinus + pad(tzOffset / 60) +
-        ':' + pad(tzOffset % 60);
-}
-
-const endpointNames = [
-    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
-    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
-    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay',
-    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
-    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
-    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
-    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
-    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
-    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
-];
-
-function capitalize(s) {
-    return s[0].toUpperCase() + s.slice(1);
-}
-
-async function getZigbee2mqttVersion() {
-    return new Promise((resolve, reject) => {
-        const git = require('git-last-commit');
-        const packageJSON = require('../../package.json');
-        const version = packageJSON.version;
-
-        git.getLastCommit((err, commit) => {
-            let commitHash = null;
-
-            if (err) {
-                try {
-                    commitHash = require('../../.hash.json').hash;
-                } catch (error) {
-                    commitHash = 'unknown';
-                }
-            } else {
-                commitHash = commit.shortHash;
-            }
-
-            resolve({commitHash, version});
-        });
-    });
-}
-
-function formatDate(date, type) {
-    let result;
-
-    switch (type) {
-    case 'ISO_8601':
-        result = new Date(date).toISOString();
-        break;
-    case 'ISO_8601_local':
-        result = toLocalISOString(new Date(date));
-        break;
-    case 'epoch':
-        result = date;
-        break;
-    case 'relative':
-        // https://github.com/EvanHahn/HumanizeDuration.js#options
-        result = humanizeDuration(Date.now() - date, {language: 'en', largest: 2, round: true}) + ' ago';
-        break;
-    default:
-        throw new Error(`Unsupported type '${type}'`);
-    }
-
-    return result;
-}
-
-function objectHasProperties(object, properties) {
-    for (const property of properties) {
-        if (!object.hasOwnProperty(property)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function equalsPartial(object, expected) {
-    for (const [key, value] of Object.entries(expected)) {
-        if (!equals(object[key], value)) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-function getObjectProperty(object, key, defaultValue) {
-    return object.hasOwnProperty(key) ? object[key] : defaultValue;
-}
-
-function getResponse(request, data, error) {
-    const response = {data, status: error ? 'error' : 'ok'};
-    if (error) response.error = error;
-    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
-        response.transaction = request.transaction;
-    }
-    return response;
-}
-
-function parseJSON(value, failedReturnValue) {
-    try {
-        return JSON.parse(value);
-    } catch (e) {
-        return failedReturnValue;
-    }
-}
-
-function* getExternalConvertersDefinitions(settings) {
-    const externalConverters = settings.get().external_converters;
-
-    for (const moduleName of externalConverters) {
-        let converterModule = moduleName;
-
-        if (moduleName.endsWith('.js')) {
-            converterModule = data.joinPath(moduleName.split('.')[0]);
-        }
-
-        const converter = require(converterModule);
-        if (Array.isArray(converter)) {
-            for (const item of converter) {
-                yield item;
-            }
-        } else {
-            yield converter;
-        }
-    }
-}
-
-function toSnakeCase(value) {
-    if (typeof value === 'object') {
-        for (const key of Object.keys(value)) {
-            const keySnakeCase = toSnakeCase(key);
-            if (key !== keySnakeCase) {
-                value[keySnakeCase] = value[key];
-                delete value[key];
-            }
-        }
-        return value;
-    } else {
-        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
-    }
-}
-
-function validateFriendlyName(name) {
-    const errors = [];
-    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
-    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
-    if (name.includes('#') || name.includes('+')) {
-        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
-    }
-    return errors;
-}
-
-module.exports = {
-    millisecondsToSeconds: (milliseconds) => milliseconds / 1000,
-    secondsToMilliseconds: (seconds) => seconds * 1000,
-    getZigbee2mqttVersion,
-    objectHasProperties,
-    toSnakeCase,
-    getObjectProperty,
-    getEndpointNames: () => endpointNames,
-    isXiaomiDevice: (device) => {
-        return device.modelID !== 'lumi.router' && xiaomiManufacturerID.includes(device.manufacturerID) &&
-            (!device.manufacturerName || !device.manufacturerName.startsWith('Trust'));
-    },
-    isIkeaTradfriDevice: (device) => ikeaTradfriManufacturerID.includes(device.manufacturerID),
-    formatDate: (date, type) => formatDate(date, type),
-    equalsPartial,
-    getResponse,
-    capitalize,
-    parseJSON,
-    getExternalConvertersDefinitions,
-    validateFriendlyName,
-};

lib/util/utils.ts

@@ -0,0 +1,290 @@
+import equals from 'fast-deep-equal/es6';
+import humanizeDuration from 'humanize-duration';
+import data from './data';
+import vm from 'vm';
+import fs from 'fs';
+import path from 'path';
+
+// construct a local ISO8601 string (instead of UTC-based)
+// Example:
+//  - ISO8601 (UTC) = 2019-03-01T15:32:45.941+0000
+//  - ISO8601 (local) = 2019-03-01T16:32:45.941+0100 (for timezone GMT+1)
+function toLocalISOString(date: Date): string {
+    const tzOffset = -date.getTimezoneOffset();
+    const plusOrMinus = tzOffset >= 0 ? '+' : '-';
+    const pad = (num: number): string => {
+        const norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+    };
+
+    return date.getFullYear() +
+        '-' + pad(date.getMonth() + 1) +
+        '-' + pad(date.getDate()) +
+        'T' + pad(date.getHours()) +
+        ':' + pad(date.getMinutes()) +
+        ':' + pad(date.getSeconds()) +
+        plusOrMinus + pad(tzOffset / 60) +
+        ':' + pad(tzOffset % 60);
+}
+
+const endpointNames = [
+    'left', 'right', 'center', 'bottom_left', 'bottom_right', 'default',
+    'top_left', 'top_right', 'white', 'rgb', 'cct', 'system', 'top', 'bottom', 'center_left', 'center_right',
+    'ep1', 'ep2', 'row_1', 'row_2', 'row_3', 'row_4', 'relay', 'usb',
+    'l1', 'l2', 'l3', 'l4', 'l5', 'l6', 'l7', 'l8',
+    'l9', 'l10', 'l11', 'l12', 'l13', 'l14', 'l15', 'l16',
+    'button_1', 'button_2', 'button_3', 'button_4', 'button_5',
+    'button_6', 'button_7', 'button_8', 'button_9', 'button_10',
+    'button_11', 'button_12', 'button_13', 'button_14', 'button_15',
+    'button_16', 'button_17', 'button_18', 'button_19', 'button_20',
+    'button_light', 'button_fan_high', 'button_fan_med', 'button_fan_low',
+    'heat', 'cool', 'water', 'meter', 'wifi',
+];
+
+function capitalize(s: string): string {
+    return s[0].toUpperCase() + s.slice(1);
+}
+
+async function getZigbee2MQTTVersion(includeCommitHash=true): Promise<{commitHash: string, version: string}> {
+    const git = await import('git-last-commit');
+    const packageJSON = await import('../..' + '/package.json');
+
+    if (!includeCommitHash) {
+        return {version: packageJSON.version, commitHash: null};
+    }
+
+    return new Promise((resolve) => {
+        const version = packageJSON.version;
+
+        git.getLastCommit((err: Error, commit: {shortHash: string}) => {
+            let commitHash = null;
+
+            if (err) {
+                try {
+                    commitHash = fs.readFileSync(path.join(__dirname, '..', '..', 'dist', '.hash'), 'utf-8');
+                } catch (error) {
+                    /* istanbul ignore next */
+                    commitHash = 'unknown';
+                }
+            } else {
+                commitHash = commit.shortHash;
+            }
+
+            resolve({commitHash, version});
+        });
+    });
+}
+
+async function getDependencyVersion(depend: string): Promise<{version: string}> {
+    const packageJSON = await import(path.join(__dirname, '..', '..', 'node_modules', depend, 'package.json'));
+    const version = packageJSON.version;
+    return {version};
+}
+
+function formatDate(time: number, type: 'ISO_8601' | 'ISO_8601_local' | 'epoch' | 'relative'): string | number {
+    if (type === 'ISO_8601') return new Date(time).toISOString();
+    else if (type === 'ISO_8601_local') return toLocalISOString(new Date(time));
+    else if (type === 'epoch') return time;
+    else { // relative
+        return humanizeDuration(Date.now() - time, {language: 'en', largest: 2, round: true}) + ' ago';
+    }
+}
+
+function objectHasProperties(object: {[s: string]: unknown}, properties: string[]): boolean {
+    for (const property of properties) {
+        if (!object.hasOwnProperty(property)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function equalsPartial(object: KeyValue, expected: KeyValue): boolean {
+    for (const [key, value] of Object.entries(expected)) {
+        if (!equals(object[key], value)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+function getObjectProperty(object: KeyValue, key: string, defaultValue: unknown): unknown {
+    return object && object.hasOwnProperty(key) ? object[key] : defaultValue;
+}
+
+function getResponse(request: KeyValue | string, data: KeyValue, error: string): MQTTResponse {
+    const response: MQTTResponse = {data, status: error ? 'error' : 'ok'};
+    if (error) response.error = error;
+    if (typeof request === 'object' && request.hasOwnProperty('transaction')) {
+        response.transaction = request.transaction;
+    }
+    return response;
+}
+
+function parseJSON(value: string, fallback: string): KeyValue | string {
+    try {
+        return JSON.parse(value);
+    } catch (e) {
+        return fallback;
+    }
+}
+
+function loadModuleFromText(moduleCode: string): unknown {
+    const moduleFakePath = path.join(__dirname, 'externally-loaded.js');
+    const sandbox = {
+        require: require,
+        module: {},
+        console,
+        setTimeout,
+        clearTimeout,
+        setInterval,
+        clearInterval,
+        setImmediate,
+        clearImmediate,
+    };
+    vm.runInNewContext(moduleCode, sandbox, moduleFakePath);
+    /* eslint-disable-line */ // @ts-ignore
+    return sandbox.module.exports;
+}
+
+function loadModuleFromFile(modulePath: string): unknown {
+    const moduleCode = fs.readFileSync(modulePath, {encoding: 'utf8'});
+    return loadModuleFromText(moduleCode);
+}
+
+function* getExternalConvertersDefinitions(settings: Settings): Generator<zhc.ExternalDefinition> {
+    const externalConverters = settings.external_converters;
+
+    for (const moduleName of externalConverters) {
+        let converter;
+
+        if (moduleName.endsWith('.js')) {
+            converter = loadModuleFromFile(data.joinPath(moduleName));
+        } else {
+            converter = require(moduleName);
+        }
+
+        if (Array.isArray(converter)) {
+            for (const item of converter) {
+                yield item;
+            }
+        } else {
+            yield converter;
+        }
+    }
+}
+
+function removeNullPropertiesFromObject(obj: KeyValue): void {
+    for (const key of Object.keys(obj)) {
+        const value = obj[key];
+        if (value == null) {
+            delete obj[key];
+        } else if (typeof value === 'object') {
+            removeNullPropertiesFromObject(value);
+        }
+    }
+}
+
+function toNetworkAddressHex(value: number): string {
+    const hex = value.toString(16);
+    return `0x${'0'.repeat(4 - hex.length)}${hex}`;
+}
+
+// eslint-disable-next-line
+function toSnakeCase(value: string | KeyValue): any {
+    if (typeof value === 'object') {
+        value = {...value};
+        for (const key of Object.keys(value)) {
+            const keySnakeCase = toSnakeCase(key);
+            if (key !== keySnakeCase) {
+                value[keySnakeCase] = value[key];
+                delete value[key];
+            }
+        }
+        return value;
+    } else {
+        return value.replace(/\.?([A-Z])/g, (x, y) => '_' + y.toLowerCase()).replace(/^_/, '').replace('_i_d', '_id');
+    }
+}
+
+function validateFriendlyName(name: string, throwFirstError=false): string[] {
+    const errors = [];
+    for (const endpointName of endpointNames) {
+        if (name.toLowerCase().endsWith('/' + endpointName)) {
+            errors.push(`friendly_name is not allowed to end with: '/${endpointName}'`);
+        }
+    }
+
+    if (name.length === 0) errors.push(`friendly_name must be at least 1 char long`);
+    if (name.endsWith('/') || name.startsWith('/')) errors.push(`friendly_name is not allowed to end or start with /`);
+    if (name.endsWith(String.fromCharCode(0))) errors.push(`friendly_name is not allowed to contain null char`);
+    if (endpointNames.includes(name)) errors.push(`Following friendly_name are not allowed: '${endpointNames}'`);
+    if (name.match(/.*\/\d*$/)) errors.push(`Friendly name cannot end with a "/DIGIT" ('${name}')`);
+    if (name.includes('#') || name.includes('+')) {
+        errors.push(`MQTT wildcard (+ and #) not allowed in friendly_name ('${name}')`);
+    }
+
+    if (throwFirstError && errors.length) {
+        throw new Error(errors[0]);
+    }
+
+    return errors;
+}
+
+function sleep(seconds: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+function sanitizeImageParameter(parameter: string): string {
+    const replaceByDash = [/\?/g, /&/g, /[^a-z\d\- _./:]/gi];
+    let sanitized = parameter;
+    replaceByDash.forEach((r) => sanitized = sanitized.replace(r, '-'));
+    return sanitized;
+}
+
+function isAvailabilityEnabledForDevice(device: Device, settings: Settings): boolean {
+    if (device.settings.hasOwnProperty('availability')) {
+        return !!device.settings.availability;
+    }
+
+    // availability_timeout = deprecated
+    const enabledGlobal = settings.advanced.availability_timeout || settings.availability;
+    if (!enabledGlobal) return false;
+
+    const passlist = settings.advanced.availability_passlist.concat(settings.advanced.availability_whitelist);
+    if (passlist.length > 0) {
+        return passlist.includes(device.name) || passlist.includes(device.ieeeAddr);
+    }
+
+    const blocklist = settings.advanced.availability_blacklist.concat(settings.advanced.availability_blocklist);
+    return !blocklist.includes(device.name) && !blocklist.includes(device.ieeeAddr);
+}
+
+const entityIDRegex = new RegExp(`^(.+?)(?:/(${endpointNames.join('|')}|\\d+))?$`);
+function parseEntityID(ID: string): {ID: string, endpoint: string} {
+    const match = ID.match(entityIDRegex);
+    return match && {ID: match[1], endpoint: match[2]};
+}
+
+function isEndpoint(obj: unknown): obj is zh.Endpoint {
+    return obj.constructor.name.toLowerCase() === 'endpoint';
+}
+
+function isZHGroup(obj: unknown): obj is zh.Group {
+    return obj.constructor.name.toLowerCase() === 'group';
+}
+
+const hours = (hours: number): number => 1000 * 60 * 60 * hours;
+const minutes = (minutes: number): number => 1000 * 60 * minutes;
+const seconds = (seconds: number): number => 1000 * seconds;
+
+
+export default {
+    endpointNames, capitalize, getZigbee2MQTTVersion, getDependencyVersion, formatDate, objectHasProperties,
+    equalsPartial, getObjectProperty, getResponse, parseJSON, loadModuleFromText, loadModuleFromFile,
+    getExternalConvertersDefinitions, removeNullPropertiesFromObject, toNetworkAddressHex, toSnakeCase,
+    parseEntityID, isEndpoint, isZHGroup, hours, minutes, seconds, validateFriendlyName, sleep,
+    sanitizeImageParameter, isAvailabilityEnabledForDevice,
+};

lib/util/yaml.js

@@ -1,36 +0,0 @@
-const yaml = require('js-yaml');
-const fs = require('fs');
-const equals = require('fast-deep-equal/es6');
-
-function read(file) {
-    try {
-        return yaml.safeLoad(fs.readFileSync(file, 'utf8'));
-    } catch (error) {
-        if (error.name === 'YAMLException') {
-            error.file = file;
-        }
-
-        throw error;
-    }
-}
-
-function readIfExists(file) {
-    return fs.existsSync(file) ? read(file) : null;
-}
-
-function writeIfChanged(file, content) {
-    const before = readIfExists(file);
-    if (!equals(before, content)) {
-        fs.writeFileSync(file, yaml.safeDump(content));
-    }
-}
-
-function updateIfChanged(file, key, value) {
-    const content = read(file);
-    if (content[key] !== value) {
-        content[key] = value;
-        writeIfChanged(file, content);
-    }
-}
-
-module.exports = {read, readIfExists, writeIfChanged, updateIfChanged};

lib/util/yaml.ts

@@ -0,0 +1,36 @@
+import yaml from 'js-yaml';
+import fs from 'fs';
+import equals from 'fast-deep-equal/es6';
+
+function read(file: string): KeyValue {
+    try {
+        return yaml.load(fs.readFileSync(file, 'utf8'));
+    } catch (error) {
+        if (error.name === 'YAMLException') {
+            error.file = file;
+        }
+
+        throw error;
+    }
+}
+
+function readIfExists(file: string, default_?: KeyValue): KeyValue {
+    return fs.existsSync(file) ? read(file) : default_;
+}
+
+function writeIfChanged(file: string, content: KeyValue): void {
+    const before = readIfExists(file);
+    if (!equals(before, content)) {
+        fs.writeFileSync(file, yaml.dump(content));
+    }
+}
+
+function updateIfChanged(file: string, key: string, value: KeyValue): void {
+    const content = read(file);
+    if (content[key] !== value) {
+        content[key] = value;
+        writeIfChanged(file, content);
+    }
+}
+
+export default {read, readIfExists, updateIfChanged, writeIfChanged};

lib/zigbee.js

@@ -1,307 +0,0 @@
-const ZigbeeHerdsman = require('zigbee-herdsman');
-const logger = require('./util/logger');
-const settings = require('./util/settings');
-const data = require('./util/data');
-const assert = require('assert');
-const utils = require('./util/utils');
-const events = require('events');
-const objectAssignDeep = require('object-assign-deep');
-const zigbeeHerdsmanConverters = require('zigbee-herdsman-converters');
-const stringify = require('json-stable-stringify-without-jsonify');
-
-const endpointNames = utils.getEndpointNames();
-const keyEndpointByNumber = new RegExp(`.*/([0-9]*)$`);
-
-class Zigbee extends events.EventEmitter {
-    constructor() {
-        super();
-        this.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler.bind(this);
-    }
-
-    async start() {
-        logger.info(`Starting zigbee-herdsman...`);
-        const herdsmanSettings = {
-            network: {
-                panID: settings.get().advanced.pan_id,
-                extendedPanID: settings.get().advanced.ext_pan_id,
-                channelList: [settings.get().advanced.channel],
-                networkKey: settings.get().advanced.network_key,
-            },
-            databasePath: data.joinPath('database.db'),
-            databaseBackupPath: data.joinPath('database.db.backup'),
-            backupPath: data.joinPath('coordinator_backup.json'),
-            serialPort: {
-                baudRate: settings.get().advanced.baudrate,
-                rtscts: settings.get().advanced.rtscts,
-                path: settings.get().serial.port,
-                adapter: settings.get().serial.adapter,
-            },
-            adapter: {
-                concurrent: settings.get().advanced.adapter_concurrent,
-            },
-        };
-
-        const herdsmanSettingsLog = objectAssignDeep.noMutate(herdsmanSettings);
-        herdsmanSettingsLog.network.networkKey = 'HIDDEN';
-        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
-
-        if (herdsmanSettings.network.networkKey === 'GENERATE') {
-            const newKey = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
-            settings.set(['advanced', 'network_key'], newKey);
-            herdsmanSettings.network.networkKey = newKey;
-        }
-
-        try {
-            herdsmanSettings.acceptJoiningDeviceHandler = this.acceptJoiningDeviceHandler;
-            this.herdsman = new ZigbeeHerdsman.Controller(herdsmanSettings);
-            await this.herdsman.start();
-        } catch (error) {
-            logger.error(`Error while starting zigbee-herdsman`);
-            throw error;
-        }
-
-        this.herdsman.on('adapterDisconnected', () => this.emit('adapterDisconnected'));
-        this.herdsman.on('deviceAnnounce', (data) => this.emit('event', 'deviceAnnounce', data));
-        this.herdsman.on('deviceInterview', (data) => this.emit('event', 'deviceInterview', data));
-        this.herdsman.on('deviceJoined', (data) => this.emit('event', 'deviceJoined', data));
-        this.herdsman.on('deviceLeave', (data) => this.emit('event', 'deviceLeave', data));
-        this.herdsman.on('message', (data) => this.emit('event', 'message', data));
-
-        logger.info('zigbee-herdsman started');
-        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
-        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
-
-        for (const device of this.getClients()) {
-            // If a passlist is used, all other device will be removed from the network.
-            const passlist = settings.get().passlist.concat(settings.get().whitelist);
-            const blocklist = settings.get().blocklist.concat(settings.get().ban);
-            if (passlist.length > 0) {
-                if (!passlist.includes(device.ieeeAddr)) {
-                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
-                    device.removeFromNetwork();
-                }
-            } else if (blocklist.includes(device.ieeeAddr)) {
-                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
-                device.removeFromNetwork();
-            }
-        }
-
-        // Check if we have to turn off the led
-        if (settings.get().serial.disable_led) {
-            await this.herdsman.setLED(false);
-        }
-
-        // Check if we have to set a transmit power
-        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
-            const transmitPower = settings.get().experimental.transmit_power;
-            await this.herdsman.setTransmitPower(transmitPower);
-            logger.info(`Set transmit power to '${transmitPower}'`);
-        }
-    }
-
-    async getCoordinatorVersion() {
-        return this.herdsman.getCoordinatorVersion();
-    }
-
-    async getNetworkParameters() {
-        return this.herdsman.getNetworkParameters();
-    }
-
-    async reset(type) {
-        await this.herdsman.reset(type);
-    }
-
-    async stop() {
-        await this.herdsman.stop();
-        logger.info('zigbee-herdsman stopped');
-    }
-
-    async permitJoin(permit) {
-        permit ?
-            logger.info('Zigbee: allowing new devices to join.') :
-            logger.info('Zigbee: disabling joining new devices.');
-
-        await this.herdsman.permitJoin(permit);
-    }
-
-    async getPermitJoin() {
-        return this.herdsman.getPermitJoin();
-    }
-
-    getClients() {
-        return this.herdsman.getDevices().filter((device) => device.type !== 'Coordinator');
-    }
-
-    getDevices() {
-        return this.herdsman.getDevices();
-    }
-
-    getDeviceByIeeeAddr(ieeeAddr) {
-        return this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
-    }
-
-    getDeviceByNetworkAddress(networkAddress) {
-        return this.herdsman.getDeviceByNetworkAddress(networkAddress);
-    }
-
-    getDevicesByType(type) {
-        return this.herdsman.getDevicesByType(type);
-    }
-
-    /**
-     * @param {string} key
-     * @return {object} {
-     *      type: device | coordinator
-     *      device|group: zigbee-herdsman entity
-     *      endpoint: selected endpoint (only if type === device)
-     *      settings: from configuration.yaml
-     *      name: name of the entity
-     *      definition: zigbee-herdsman-converters definition (only if type === device)
-     * }
-     */
-    resolveEntity(key) {
-        assert(
-            typeof key === 'string' || typeof key === 'number' ||
-            key.constructor.name === 'Device' || key.constructor.name === 'Group',
-            `Wrong type '${typeof key}'`,
-        );
-
-        const deviceOptions = settings.get().device_options;
-        if (typeof key === 'string' || typeof key === 'number') {
-            if (typeof key === 'number') {
-                key = key.toString();
-            }
-
-            if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
-                const coordinator = this.getDevicesByType('Coordinator')[0];
-                return {
-                    type: 'device',
-                    device: coordinator,
-                    endpoint: coordinator.getEndpoint(1),
-                    settings: {friendlyName: 'Coordinator'},
-                    name: 'Coordinator',
-                };
-            }
-
-            let endpointKey = endpointNames.find((p) => key.endsWith(`/${p}`));
-            const endpointByNumber = key.match(keyEndpointByNumber);
-            if (!endpointKey && endpointByNumber) {
-                endpointKey = Number(endpointByNumber[1]);
-            }
-            if (endpointKey) {
-                key = key.replace(`/${endpointKey}`, '');
-            }
-
-            const entity = settings.getEntity(key);
-            if (!entity) {
-                return null;
-            } else if (entity.type === 'device') {
-                const device = this.getDeviceByIeeeAddr(entity.ID);
-                if (!device) {
-                    return null;
-                }
-
-                const definition = zigbeeHerdsmanConverters.findByDevice(device);
-                const endpoints = definition && definition.endpoint ? definition.endpoint(device) : null;
-                let endpoint;
-                if (endpointKey) {
-                    if (endpointByNumber) {
-                        endpoint = device.getEndpoint(endpointKey);
-                    } else {
-                        assert(definition != null, `Endpoint name '${endpointKey}' is given but device is unsupported`);
-                        assert(endpoints != null, `Endpoint name '${endpointKey}' is given but no endpoints defined`);
-                        const endpointID = endpoints[endpointKey];
-                        assert(endpointID, `Endpoint name '${endpointKey}' is given but device has no such endpoint`);
-                        endpoint = device.getEndpoint(endpointID);
-                    }
-                } else if (endpoints && endpoints['default']) {
-                    endpoint = device.getEndpoint(endpoints['default']);
-                } else {
-                    endpoint = device.endpoints[0];
-                }
-
-                return {
-                    type: 'device', device, endpoint, settings: {...deviceOptions, ...entity},
-                    name: entity.friendlyName, definition,
-                };
-            } else {
-                let group = this.getGroupByID(entity.ID);
-                if (!group) group = this.createGroup(entity.ID);
-                return {type: 'group', group, settings: {...deviceOptions, ...entity}, name: entity.friendlyName};
-            }
-        } else if (key.constructor.name === 'Device') {
-            const setting = settings.getEntity(key.ieeeAddr);
-            return {
-                type: 'device',
-                device: key,
-                endpoint: key.endpoints[0],
-                settings: {...deviceOptions, ...(setting || {})},
-                name: setting ? setting.friendlyName : (key.type === 'Coordinator' ? 'Coordinator' : key.ieeeAddr),
-                definition: zigbeeHerdsmanConverters.findByDevice(key),
-            };
-        } else { // Group
-            const setting = settings.getEntity(key.groupID);
-            return {
-                type: 'group',
-                group: key,
-                settings: {...deviceOptions, ...(setting || {})},
-                name: setting ? setting.friendlyName : key.groupID,
-            };
-        }
-    }
-
-    getGroupByID(ID) {
-        return this.herdsman.getGroupByID(ID);
-    }
-
-    getGroups() {
-        return this.herdsman.getGroups();
-    }
-
-    createGroup(groupID) {
-        return this.herdsman.createGroup(groupID);
-    }
-
-    acceptJoiningDeviceHandler(ieeeAddr) {
-        // If passlist is set, all devices not on passlist will be rejected to join the network
-        const passlist = settings.get().passlist.concat(settings.get().whitelist);
-        const blocklist = settings.get().blocklist.concat(settings.get().ban);
-        if (passlist.length > 0) {
-            if (passlist.includes(ieeeAddr)) {
-                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
-                return true;
-            } else {
-                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
-                return false;
-            }
-        } else if (blocklist.length > 0) {
-            if (blocklist.includes(ieeeAddr)) {
-                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
-                return false;
-            } else {
-                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
-                return true;
-            }
-        } else {
-            return true;
-        }
-    }
-
-    async touchlinkFactoryResetFirst() {
-        return this.herdsman.touchlinkFactoryResetFirst();
-    }
-
-    async touchlinkFactoryReset(ieeeAddr, channel) {
-        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
-    }
-
-    async touchlinkIdentify(ieeeAddr, channel) {
-        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
-    }
-
-    async touchlinkScan() {
-        return this.herdsman.touchlinkScan();
-    }
-}
-
-module.exports = Zigbee;

lib/zigbee.ts

@@ -0,0 +1,330 @@
+import {Controller} from 'zigbee-herdsman';
+import logger from './util/logger';
+import * as settings from './util/settings';
+import data from './util/data';
+import utils from './util/utils';
+import objectAssignDeep from 'object-assign-deep';
+import stringify from 'json-stable-stringify-without-jsonify';
+import Device from './model/device';
+import Group from './model/group';
+import * as ZHEvents from 'zigbee-herdsman/dist/controller/events';
+import bind from 'bind-decorator';
+
+export default class Zigbee {
+    private herdsman: Controller;
+    private eventBus: EventBus;
+    private groupLookup: {[s: number]: Group} = {};
+    private deviceLookup: {[s: string]: Device} = {};
+
+    constructor(eventBus: EventBus) {
+        this.eventBus = eventBus;
+    }
+
+    async start(): Promise<'reset' | 'resumed' | 'restored'> {
+        const infoHerdsman = await utils.getDependencyVersion('zigbee-herdsman');
+        logger.info(`Starting zigbee-herdsman (${infoHerdsman.version})`);
+        const herdsmanSettings = {
+            network: {
+                panID: settings.get().advanced.pan_id === 'GENERATE' ?
+                    this.generatePanID() : settings.get().advanced.pan_id as number,
+                extendedPanID: settings.get().advanced.ext_pan_id,
+                channelList: [settings.get().advanced.channel],
+                networkKey: settings.get().advanced.network_key === 'GENERATE' ?
+                    this.generateNetworkKey() : settings.get().advanced.network_key as number[],
+            },
+            databasePath: data.joinPath('database.db'),
+            databaseBackupPath: data.joinPath('database.db.backup'),
+            backupPath: data.joinPath('coordinator_backup.json'),
+            serialPort: {
+                baudRate: settings.get().advanced.baudrate,
+                rtscts: settings.get().advanced.rtscts,
+                path: settings.get().serial.port,
+                adapter: settings.get().serial.adapter,
+            },
+            adapter: {
+                concurrent: settings.get().advanced.adapter_concurrent,
+                delay: settings.get().advanced.adapter_delay,
+                disableLED: settings.get().serial.disable_led,
+            },
+            acceptJoiningDeviceHandler: this.acceptJoiningDeviceHandler,
+        };
+
+        const herdsmanSettingsLog = objectAssignDeep({}, herdsmanSettings, {network: {networkKey: 'HIDDEN'}});
+        logger.debug(`Using zigbee-herdsman with settings: '${stringify(herdsmanSettingsLog)}'`);
+
+        let startResult;
+        try {
+            this.herdsman = new Controller(herdsmanSettings, logger);
+            startResult = await this.herdsman.start();
+        } catch (error) {
+            logger.error(`Error while starting zigbee-herdsman`);
+            throw error;
+        }
+
+        this.herdsman.on('adapterDisconnected', () => this.eventBus.emitAdapterDisconnected());
+        this.herdsman.on('lastSeenChanged', (data: ZHEvents.LastSeenChangedPayload) => {
+            this.eventBus.emitLastSeenChanged({device: this.resolveDevice(data.device.ieeeAddr)});
+        });
+        this.herdsman.on('permitJoinChanged', (data: ZHEvents.PermitJoinChangedPayload) => {
+            this.eventBus.emitPermitJoinChanged(data);
+        });
+        this.herdsman.on('deviceNetworkAddressChanged', (data: ZHEvents.DeviceNetworkAddressChangedPayload) => {
+            const device = this.resolveDevice(data.device.ieeeAddr);
+            logger.debug(`Device '${device.name}' changed network address`);
+            this.eventBus.emitDeviceNetworkAddressChanged({device});
+        });
+        this.herdsman.on('deviceAnnounce', (data: ZHEvents.DeviceAnnouncePayload) => {
+            const device = this.resolveDevice(data.device.ieeeAddr);
+            logger.debug(`Device '${device.name}' announced itself`);
+            this.eventBus.emitDeviceAnnounce({device});
+        });
+        this.herdsman.on('deviceInterview', (data: ZHEvents.DeviceInterviewPayload) => {
+            const device = this.resolveDevice(data.device.ieeeAddr);
+            const d = {device, status: data.status};
+            this.logDeviceInterview(d);
+            this.eventBus.emitDeviceInterview(d);
+        });
+        this.herdsman.on('deviceJoined', (data: ZHEvents.DeviceJoinedPayload) => {
+            const device = this.resolveDevice(data.device.ieeeAddr);
+            logger.info(`Device '${device.name}' joined`);
+            this.eventBus.emitDeviceJoined({device});
+        });
+        this.herdsman.on('deviceLeave', (data: ZHEvents.DeviceLeavePayload) => {
+            const name = settings.getDevice(data.ieeeAddr)?.friendly_name || data.ieeeAddr;
+            logger.warn(`Device '${name}' left the network`);
+            this.eventBus.emitDeviceLeave({ieeeAddr: data.ieeeAddr, name});
+        });
+        this.herdsman.on('message', (data: ZHEvents.MessagePayload) => {
+            const device = this.resolveDevice(data.device.ieeeAddr);
+            logger.debug(`Received Zigbee message from '${device.name}', type '${data.type}', ` +
+                `cluster '${data.cluster}', data '${stringify(data.data)}' from endpoint ${data.endpoint.ID}` +
+                (data.hasOwnProperty('groupID') ? ` with groupID ${data.groupID}` : ``));
+            this.eventBus.emitDeviceMessage({...data, device});
+        });
+
+        logger.info(`zigbee-herdsman started (${startResult})`);
+        logger.info(`Coordinator firmware version: '${stringify(await this.getCoordinatorVersion())}'`);
+        logger.debug(`Zigbee network parameters: ${stringify(await this.herdsman.getNetworkParameters())}`);
+
+        for (const device of this.devices(false)) {
+            // If a passlist is used, all other device will be removed from the network.
+            const passlist = settings.get().passlist.concat(settings.get().whitelist);
+            const blocklist = settings.get().blocklist.concat(settings.get().ban);
+            const remove = async (device: Device): Promise<void> => {
+                try {
+                    await device.zh.removeFromNetwork();
+                } catch (error) {
+                    logger.error(`Failed to remove '${device.ieeeAddr}' (${error.message})`);
+                }
+            };
+            if (passlist.length > 0) {
+                if (!passlist.includes(device.ieeeAddr)) {
+                    logger.warn(`Device which is not on passlist connected (${device.ieeeAddr}), removing...`);
+                    await remove(device);
+                }
+            } else if (blocklist.includes(device.ieeeAddr)) {
+                logger.warn(`Device on blocklist is connected (${device.ieeeAddr}), removing...`);
+                await remove(device);
+            }
+        }
+
+        // Check if we have to turn off the led
+        if (settings.get().serial.disable_led) {
+            await this.herdsman.setLED(false);
+        }
+
+        // Check if we have to set a transmit power
+        if (settings.get().experimental.hasOwnProperty('transmit_power')) {
+            const transmitPower = settings.get().experimental.transmit_power;
+            await this.herdsman.setTransmitPower(transmitPower);
+            logger.info(`Set transmit power to '${transmitPower}'`);
+        }
+
+        return startResult;
+    }
+
+    private logDeviceInterview(data: eventdata.DeviceInterview): void {
+        const name = data.device.name;
+        if (data.status === 'successful') {
+            logger.info(`Successfully interviewed '${name}', device has successfully been paired`);
+
+            if (data.device.definition) {
+                const {vendor, description, model} = data.device.definition;
+                logger.info(`Device '${name}' is supported, identified as: ${vendor} ${description} (${model})`);
+            } else {
+                logger.warn(`Device '${name}' with Zigbee model '${data.device.zh.modelID}' and manufacturer name ` +
+                    `'${data.device.zh.manufacturerName}' is NOT supported, ` +
+                    `please follow https://www.zigbee2mqtt.io/how_tos/how_to_support_new_devices.html`);
+            }
+        } else if (data.status === 'failed') {
+            logger.error(`Failed to interview '${name}', device has not successfully been paired`);
+        } else { // data.status === 'started'
+            logger.info(`Starting interview of '${name}'`);
+        }
+    }
+
+    private generateNetworkKey(): number[] {
+        const key = Array.from({length: 16}, () => Math.floor(Math.random() * 255));
+        settings.set(['advanced', 'network_key'], key);
+        return key;
+    }
+
+    private generatePanID(): number {
+        const panID = Math.floor(Math.random() * (0xFFFF - 2)) + 1;
+        settings.set(['advanced', 'pan_id'], panID);
+        return panID;
+    }
+
+    async getCoordinatorVersion(): Promise<zh.CoordinatorVersion> {
+        return this.herdsman.getCoordinatorVersion();
+    }
+
+    isStopping(): boolean {
+        return this.herdsman.isStopping();
+    }
+
+    async getNetworkParameters(): Promise<zh.NetworkParameters> {
+        return this.herdsman.getNetworkParameters();
+    }
+
+    async reset(type: 'soft' | 'hard'): Promise<void> {
+        await this.herdsman.reset(type);
+    }
+
+    async stop(): Promise<void> {
+        logger.info('Stopping zigbee-herdsman...');
+        await this.herdsman.stop();
+        logger.info('Stopped zigbee-herdsman');
+    }
+
+    getPermitJoin(): boolean {
+        return this.herdsman.getPermitJoin();
+    }
+
+    getPermitJoinTimeout(): number {
+        return this.herdsman.getPermitJoinTimeout();
+    }
+
+    async permitJoin(permit: boolean, device?: Device, time: number=undefined): Promise<void> {
+        if (permit) {
+            logger.info(`Zigbee: allowing new devices to join${device ? ` via ${device.name}` : ''}.`);
+        } else {
+            logger.info('Zigbee: disabling joining new devices.');
+        }
+
+        if (device && permit) {
+            await this.herdsman.permitJoin(permit, device.zh, time);
+        } else {
+            await this.herdsman.permitJoin(permit, undefined, time);
+        }
+    }
+
+    private resolveDevice(ieeeAddr: string): Device {
+        if (!this.deviceLookup[ieeeAddr]) {
+            const device = this.herdsman.getDeviceByIeeeAddr(ieeeAddr);
+            device && (this.deviceLookup[ieeeAddr] = new Device(device));
+        }
+
+        const device = this.deviceLookup[ieeeAddr];
+        if (device && !device.zh.isDeleted) {
+            device.ensureInSettings();
+            return device;
+        }
+    }
+
+    private resolveGroup(groupID: number): Group {
+        const group = this.herdsman.getGroupByID(Number(groupID));
+        if (group && !this.groupLookup[groupID]) {
+            this.groupLookup[groupID] = new Group(group);
+        }
+
+        return this.groupLookup[groupID];
+    }
+
+    resolveEntity(key: string | number | zh.Device): Device | Group {
+        if (typeof key === 'object') {
+            return this.resolveDevice(key.ieeeAddr);
+        } else if (typeof key === 'string' && key.toLowerCase() === 'coordinator') {
+            return this.resolveDevice(this.herdsman.getDevicesByType('Coordinator')[0].ieeeAddr);
+        } else {
+            const settingsDevice = settings.getDevice(key.toString());
+            if (settingsDevice) return this.resolveDevice(settingsDevice.ID);
+
+            const groupSettings = settings.getGroup(key);
+            if (groupSettings) {
+                const group = this.resolveGroup(groupSettings.ID);
+                // If group does not exist, create it (since it's already in configuration.yaml)
+                return group ? group : this.createGroup(groupSettings.ID);
+            }
+        }
+    }
+
+    firstCoordinatorEndpoint(): zh.Endpoint {
+        return this.herdsman.getDevicesByType('Coordinator')[0].endpoints[0];
+    }
+
+    groups(): Group[] {
+        return this.herdsman.getGroups().map((g) => this.resolveGroup(g.groupID));
+    }
+
+    devices(includeCoordinator=true): Device[] {
+        return this.herdsman.getDevices()
+            .map((d) => this.resolveDevice(d.ieeeAddr))
+            .filter((d) => includeCoordinator || d.zh.type !== 'Coordinator');
+    }
+
+    @bind private async acceptJoiningDeviceHandler(ieeeAddr: string): Promise<boolean> {
+        // If passlist is set, all devices not on passlist will be rejected to join the network
+        const passlist = settings.get().passlist.concat(settings.get().whitelist);
+        const blocklist = settings.get().blocklist.concat(settings.get().ban);
+        if (passlist.length > 0) {
+            if (passlist.includes(ieeeAddr)) {
+                logger.info(`Accepting joining device which is on passlist '${ieeeAddr}'`);
+                return true;
+            } else {
+                logger.info(`Rejecting joining not in passlist device '${ieeeAddr}'`);
+                return false;
+            }
+        } else if (blocklist.length > 0) {
+            if (blocklist.includes(ieeeAddr)) {
+                logger.info(`Rejecting joining device which is on blocklist '${ieeeAddr}'`);
+                return false;
+            } else {
+                logger.info(`Accepting joining not in blocklist device '${ieeeAddr}'`);
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    async touchlinkFactoryResetFirst(): Promise<boolean> {
+        return this.herdsman.touchlinkFactoryResetFirst();
+    }
+
+    async touchlinkFactoryReset(ieeeAddr: string, channel: number): Promise<boolean> {
+        return this.herdsman.touchlinkFactoryReset(ieeeAddr, channel);
+    }
+
+    async touchlinkIdentify(ieeeAddr: string, channel: number): Promise<void> {
+        await this.herdsman.touchlinkIdentify(ieeeAddr, channel);
+    }
+
+    async touchlinkScan(): Promise<{ieeeAddr: string, channel: number}[]> {
+        return this.herdsman.touchlinkScan();
+    }
+
+    createGroup(ID: number): Group {
+        this.herdsman.createGroup(ID);
+        return this.resolveGroup(ID);
+    }
+
+    deviceByNetworkAddress(networkAddress: number): Device {
+        const device = this.herdsman.getDeviceByNetworkAddress(networkAddress);
+        return device && this.resolveDevice(device.ieeeAddr);
+    }
+
+    groupByID(ID: number): Group {
+        return this.resolveGroup(ID);
+    }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "zigbee2mqtt",
-  "version": "1.15.0",
+  "version": "1.22.0",
   "description": "Zigbee to MQTT bridge using Zigbee-herdsman",
   "main": "index.js",
   "repository": {
@@ -8,7 +8,7 @@
     "url": "git+https://github.com/Koenkk/zigbee2mqtt.git"
   },
   "engines": {
-    "node": "^10 || ^12 || ^14"
+    "node": "^10 || ^12 || ^14 || ^15 || ^16"
   },
   "keywords": [
     "xiaomi",
@@ -20,9 +20,11 @@
     "cc2531"
   ],
   "scripts": {
-    "test-with-coverage": "jest test --coverage",
-    "eslint": "node_modules/.bin/eslint lib/",
+    "build": "tsc && node index.js writehash",
+    "build-watch": "tsc --watch",
+    "eslint": "eslint lib/ --max-warnings=0",
     "start": "node index.js",
+    "test-with-coverage": "jest test --coverage",
     "test": "jest test",
     "test-watch": "jest test --watch"
   },
@@ -33,35 +35,55 @@
   },
   "homepage": "https://koenkk.github.io/zigbee2mqtt",
   "dependencies": {
-    "ajv": "^6.12.4",
-    "debounce": "^1.2.0",
+    "ajv": "^8.6.3",
+    "bind-decorator": "^1.0.11",
+    "core-js": "^3.18.2",
+    "debounce": "^1.2.1",
+    "deep-object-diff": "^1.1.0",
     "fast-deep-equal": "^3.1.3",
     "finalhandler": "^1.1.2",
-    "git-last-commit": "^1.0.0",
-    "http-proxy": "^1.18.1",
-    "humanize-duration": "^3.23.1",
-    "js-yaml": "^3.14.0",
-    "json-stable-stringify-without-jsonify": "=1.0.1",
+    "git-last-commit": "^1.0.1",
+    "humanize-duration": "^3.27.0",
+    "js-yaml": "^4.1.0",
+    "json-stable-stringify-without-jsonify": "^1.0.1",
     "mkdir-recursive": "^0.4.0",
-    "moment": "^2.27.0",
-    "mqtt": "^4.2.1",
+    "moment": "^2.29.1",
+    "mqtt": "4.2.8",
     "object-assign-deep": "^0.4.0",
     "rimraf": "^3.0.2",
-    "semver": "^7.3.2",
+    "semver": "^7.3.5",
     "serve-static": "^1.14.1",
+    "source-map-support": "^0.5.20",
     "winston": "^3.3.3",
     "winston-syslog": "^2.4.4",
-    "ws": "^7.3.1",
-    "zigbee-herdsman": "0.13.11",
-    "zigbee-herdsman-converters": "12.0.194",
-    "zigbee2mqtt-frontend": "0.2.17"
+    "ws": "^8.2.3",
+    "zigbee-herdsman": "0.13.164",
+    "zigbee-herdsman-converters": "14.0.303",
+    "zigbee2mqtt-frontend": "0.6.30"
   },
   "devDependencies": {
-    "eslint": "*",
-    "eslint-config-google": "*",
-    "eslint-plugin-jest": "*",
-    "jest": "*",
-    "tmp": "*"
+    "@babel/core": "^7.15.5",
+    "@babel/plugin-proposal-decorators": "^7.15.4",
+    "@babel/preset-env": "^7.15.6",
+    "@babel/preset-typescript": "^7.15.0",
+    "@types/debounce": "^1.2.1",
+    "@types/finalhandler": "^1.1.1",
+    "@types/humanize-duration": "^3.25.1",
+    "@types/jest": "^27.0.2",
+    "@types/js-yaml": "^4.0.3",
+    "@types/object-assign-deep": "^0.4.0",
+    "@types/rimraf": "^3.0.2",
+    "@types/serve-static": "^1.13.10",
+    "@types/ws": "^8.2.0",
+    "@typescript-eslint/eslint-plugin": "^4.32.0",
+    "@typescript-eslint/parser": "^4.32.0",
+    "babel-jest": "^27.2.4",
+    "eslint": "^7.32.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-jest": "^24.5.0",
+    "jest": "^27.2.4",
+    "tmp": "^0.2.1",
+    "typescript": "^4.4.3"
   },
   "jest": {
     "coverageThreshold": {
@@ -73,7 +95,11 @@
       }
     },
     "collectCoverageFrom": [
-      "lib/**"
+      "lib/**/*.js",
+      "lib/**/*.ts"
     ]
+  },
+  "bin": {
+    "zigbee2mqtt": "cli.js"
   }
 }

scripts/zStackEraseAllNvMem.js

@@ -0,0 +1,89 @@
+const {ZnpCommandStatus, NvSystemIds} = require('zigbee-herdsman/dist/adapter/z-stack/constants/common');
+const {ZnpVersion} = require('zigbee-herdsman/dist/adapter/z-stack/adapter/tstype');
+const {Subsystem} = require('zigbee-herdsman/dist/adapter/z-stack/unpi/constants');
+const {Znp} = require('zigbee-herdsman/dist/adapter/z-stack/znp');
+
+
+class ZStackNvMemEraser {
+    constructor(device) {
+        this.znp = new Znp(device, 115200, false);
+    }
+
+    async start() {
+        await this.znp.open();
+        const attempts = 3;
+        for (let i = 0; i < attempts; i++) {
+            try {
+                await this.znp.request(Subsystem.SYS, 'ping', {capabilities: 1});
+                break;
+            } catch (e) {
+                if (attempts - 1 === i) {
+                    throw new Error(`Failed to connect to the adapter (${e})`);
+                }
+            }
+        }
+        // Old firmware did not support version, assume it's Z-Stack 1.2 for now.
+        try {
+            this.version = (await this.znp.request(Subsystem.SYS, 'version', {})).payload;
+        } catch (e) {
+            console.log(`Failed to get zStack version, assuming 1.2`);
+            this.version = {'transportrev': 2, 'product': 0, 'majorrel': 2,
+                'minorrel': 0, 'maintrel': 0, 'revision': ''};
+        }
+
+        console.log(`Detected znp version '${ZnpVersion[this.version.product]}' (${JSON.stringify(this.version)})`);
+
+        await this.clearAllNvMemItems();
+
+        process.exit(0);
+    }
+
+    async clearAllNvMemItems() {
+        let maxNvMemId;
+        switch (this.version.product) {
+        case ZnpVersion.zStack12: maxNvMemId = 0x0302; break;
+        case ZnpVersion.zStack30x: maxNvMemId = 0x033F; break;
+        case ZnpVersion.zStack3x0: maxNvMemId = 0x032F; break;
+        }
+
+        let deletedCount = 0;
+        console.log(`Clearing all NVMEM items, from 0 to ${maxNvMemId}`);
+        for (let id=0; id<=maxNvMemId; id++) {
+            let len;
+            const needOsal = !(this.version.product == ZnpVersion.zStack3x0 && id <= 7);
+            if (needOsal) {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'osalNvLength', {id: id});
+                len = lengthRes.payload['length'];
+            } else {
+                const lengthRes = await this.znp.request(Subsystem.SYS, 'nvLength',
+                    {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0});
+                len = lengthRes.payload['len'];
+            }
+            if (len != 0) {
+                console.log(`NVMEM item #${id} - deleting, size: ${len}`);
+                if (needOsal) {
+                    await this.znp.request(Subsystem.SYS, 'osalNvDelete',
+                        {id: id, len: len},
+                        null, null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                } else {
+                    await this.znp.request(Subsystem.SYS, 'nvDelete',
+                        {sysid: NvSystemIds.ZSTACK, itemid: id, subid: 0},
+                        null, null, [ZnpCommandStatus.SUCCESS, ZnpCommandStatus.NV_ITEM_INITIALIZED]);
+                }
+                deletedCount++;
+            }
+        }
+        console.log(`Clearing all NVMEM items finished, deleted ${deletedCount} items`);
+    }
+}
+const processArgs = process.argv.slice(2);
+if (processArgs.length != 1) {
+    console.log('ZStack NVMEM eraser.');
+    console.log('Usage:');
+    console.log('   node zStackEraseAllNvMem.js <SERIAL_PORT>');
+    process.exit(1);
+}
+
+const eraser = new ZStackNvMemEraser(processArgs[0]);
+
+eraser.start();

scripts/zigbee2socat_installer.sh

@@ -53,7 +53,7 @@ function handle_opts {
 
         -v|--verbose)
             verbose=1; shift; continue ;;
-        
+
         -m|--man)
             makemanual ;;
 
@@ -84,7 +84,6 @@ port=""
 function main {
     handle_opts "$@"
 
-    # TODO: Check IP-address valid?
     if [[ "$ip" = "" ]]; then
         echo "IP-address not specified"
         exit 5
@@ -161,7 +160,7 @@ function usage {
     -v, --verbose       Print more information
     -m, --man           Make and display manual
     -h, --help          Display this help message
-    
+
     \rOriginal concept by JFLN\
     "
     exit 0

test/assets/exampleExtension.js

@@ -1,9 +1,10 @@
 class Example {
     constructor(zigbee, mqtt, state, publishEntityState, eventBus) {
         this.mqtt = mqtt;
+        this.mqtt.publish('example/extension', 'call from constructor')
     }
 
-    onMQTTConnected() {
+    start() {
         this.mqtt.publish('example/extension', 'test')
     }
 }

test/assets/mock-external-converter-multiple.js

@@ -11,10 +11,23 @@ const homeassistantSwitch = {
 
 const mockDevices = [{
     mock: 1,
-    model: 'external_converters_device',
+    model: 'external_converters_device_1',
     homeassistant: [homeassistantSwitch],
+    zigbeeModel: ['external_converter_device_1'],
+    vendor: 'external_1',
+    description: 'external_1',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 }, {
-    mock: 2
+    mock: 2,
+    model: 'external_converters_device_2',
+    zigbeeModel: ['external_converter_device_2'],
+    vendor: 'external_2',
+    description: 'external_2',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 }];
 
 module.exports = mockDevices;

test/assets/mock-external-converter.js

@@ -1,5 +1,12 @@
 const mockDevice = {
-    mock: true
+    mock: true,
+    zigbeeModel: ['external_converter_device'],
+    vendor: 'external',
+    model: 'external_converter_device',
+    description: 'external',
+    fromZigbee: [],
+    toZigbee: [],
+    exposes: [],
 };
 
 module.exports = mockDevice;

test/availability.test.js

@@ -1,412 +1,285 @@
-const data = require('./stub/data');
-const logger = require('./stub/logger');
-const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
-zigbeeHerdsman.returnDevices.push('0x000b57fffec6a5b3');
-zigbeeHerdsman.returnDevices.push('0x00124b00120144ae');
-zigbeeHerdsman.returnDevices.push('0x0017880104e45553');
-zigbeeHerdsman.returnDevices.push('0x0017880104e45517');
+import data from './stub/data';
+import logger from './stub/logger';
+import MQTT from './stub/mqtt';
+import zigbeeHerdsman from './stub/zigbeeHerdsman';
 
-const MQTT = require('./stub/mqtt');
-const settings = require('../lib/util/settings');
-const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
-const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+import utils from '../lib/util/utils';
+import * as settings from '../lib/util/settings';
+import Controller from '../lib/controller';
+import flushPromises from './lib/flushPromises';
+import stringify from 'json-stable-stringify-without-jsonify';
 
-const mocksClear = [MQTT.publish, logger.warn, logger.debug];
+const mocks = [MQTT.publish, logger.warn, logger.debug];
+const devices = zigbeeHerdsman.devices;
+zigbeeHerdsman.returnDevices.push(
+    ...[devices.bulb_color.ieeeAddr, devices.bulb_color_2.ieeeAddr, devices.coordinator.ieeeAddr, devices.remote.ieeeAddr])
 
 describe('Availability', () => {
     let controller;
 
-    function getExtension() {
-        return controller.extensions.find((e) => e.constructor.name === 'Availability');
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Availability');
+        await controller.enableDisableExtension(true, 'Availability');
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        jest.useFakeTimers();
-        settings.set(['advanced', 'availability_timeout'], 10);
-        settings.set(['experimental', 'new_api'], true);
-        controller = new Controller();
-        mocksClear.forEach((m) => m.mockClear());
+    const advancedTime = async (value) => {
+        jest.setSystemTime(Date.now() + value);
+        jest.advanceTimersByTime(value);
+        await flushPromises();
+    }
+
+    beforeAll(async () => {
+        jest.spyOn(utils, 'sleep').mockImplementation(async (seconds) => {});
+        jest.useFakeTimers('modern');
+        settings.reRead();
+        settings.set(['availability'], true);
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
     });
 
+    beforeEach(async () => {
+        jest.useFakeTimers('modern').setSystemTime(utils.minutes(1));
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        settings.set(['availability'], true);
+        settings.set(['devices', devices.bulb_color_2.ieeeAddr, 'availability'], false);
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
+        mocks.forEach((m) => m.mockClear());
+        await resetExtension();
+        Object.values(devices).forEach((d) => d.ping.mockClear());
+    });
+
     afterEach(async () => {
-        await controller.stop();
-        await flushPromises();
+        Object.values(devices).forEach(d => d.lastSeen = utils.minutes(1));
     })
 
-    it('Should publish availabilty on startup', async () => {
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
-          'online',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should publish availabilty on startup for device where it is enabled for', async () => {
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        expect(MQTT.publish).not.toHaveBeenCalledWith('zigbee2mqtt/bulb_color_2/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
     });
 
-    it('Should publish availabilty offline when ping fails', async () => {
+    it('Should publish offline for active device when not seen for 10 minutes', async () => {
         MQTT.publish.mockClear();
-        logger.error.mockClear();
-        logger.debug.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
-        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
-            'zigbee2mqtt/bulb_color/availability',
-          'offline',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
-        expect(logger.error).toHaveBeenCalledTimes(1);
-        expect(logger.error).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
-        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(logger.debug).toHaveBeenCalledTimes(3);
-        expect(logger.debug).toHaveBeenCalledWith("Failed to ping 'bulb_color'");
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
-            'zigbee2mqtt/bulb_color/availability',
-          'offline',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
     });
 
-    it('Should publish availabilty online and query state on reconnect', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
-        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
+    it('Shouldnt do anything for a device when availability: false is set for device', async () => {
         MQTT.publish.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color_2}); // Coverage satisfaction
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should publish offline for passive device when not seen for 25 hours', async () => {
+        MQTT.publish.mockClear();
+        await advancedTime(utils.hours(26));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for active device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should ping again when first ping fails', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(5));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        devices.bulb_color.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        devices.bulb_color.lastSeen = Date.now() + utils.minutes(10);
+        await advancedTime(utils.minutes(10));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(2);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(1, true);
+        expect(devices.bulb_color.ping).toHaveBeenNthCalledWith(2, false);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should reset ping timer when device last seen changes for passive device', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.hours(24));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.remote});
+
+        await advancedTime(utils.hours(25));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+
+        devices.remote.ping.mockImplementationOnce(() => {throw new Error('failed')});
+        await advancedTime(utils.hours(3));
+        expect(devices.remote.ping).toHaveBeenCalledTimes(0);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should immediately mark device as online when it lastSeen changes', async () => {
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(15));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
+
+        devices.bulb_color.lastSeen = Date.now();
+        await zigbeeHerdsman.events.lastSeenChanged({device: devices.bulb_color});
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
-          'online',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_color/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+    });
+
+    it('Should allow to change availability timeout via device options', async () => {
+        settings.set(['devices', '0x000b57fffec6a5b3', 'availability'], {timeout: 40});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(17));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should allow to change availability timeout via avaiability options', async () => {
+        settings.set(['availability'], {active: {timeout: 30}});
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(25));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await advancedTime(utils.minutes(7));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
+    });
+
+    it('Should stop pinging device when it leaves', async () => {
+        await resetExtension();
+        MQTT.publish.mockClear();
+
+        await advancedTime(utils.minutes(9));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+
+        await zigbeeHerdsman.events.deviceLeave({ieeeAddr: devices.bulb_color.ieeeAddr});
+        await flushPromises();
+
+        await advancedTime(utils.minutes(3));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should allow to be disabled', async () => {
+        settings.set(['availability'], false);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+    });
+
+    it('Should retrieve device state when it reconnects', async () => {
+        MQTT.publish.mockClear();
+
+        const endpoint = devices.bulb_color.getEndpoint(1);
+        endpoint.read.mockClear();
+
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+        await advancedTime(utils.seconds(1));
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
+        await flushPromises();
+
+        expect(endpoint.read).toHaveBeenCalledTimes(0);
+        await advancedTime(utils.seconds(2));
 
         expect(endpoint.read).toHaveBeenCalledTimes(3);
-        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
         expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
-    });
+        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
+        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl',
+            ['colorMode', 'currentX', 'currentY', 'enhancedCurrentHue', 'currentSaturation', 'colorTemperature']);
 
-    it('Should fail gracefully when quering state after reconnect fails', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
+        // Should stop when one request fails
         endpoint.read.mockClear();
-        endpoint.read.mockImplementationOnce(() => {throw new Error('Device timedout')});
-        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
-        logger.debug.mockClear();
-        jest.advanceTimersByTime(11 * 2000);
+        await zigbeeHerdsman.events.deviceAnnounce({device: devices.bulb_color});
         await flushPromises();
-
+        endpoint.read.mockImplementationOnce(() => {throw new Error('')});
+        await advancedTime(utils.seconds(3));
         expect(endpoint.read).toHaveBeenCalledTimes(1);
-        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(logger.error).toHaveBeenCalledWith(`Failed to read state of 'bulb_color' after reconnect`);
     });
 
-    it('Shouldnt ping again when still pinging', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        device.ping.mockClear();
-        device.ping.mockImplementationOnce(async () => {await wait(100000000)});
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should mark device online when receiving message while offline', async () => {
+    it('Should republish availability when device is renamed', async () => {
         MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
-        device.ping.mockImplementationOnce(() => {throw new Error('failed')});
-        jest.advanceTimersByTime(11 * 1000);
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/rename', stringify({from: 'bulb_color', to: 'bulb_new_name'}));
         await flushPromises();
 
-        MQTT.publish.mockClear();
-        const data = {modelID: 'test'}
-        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenNthCalledWith(1,
-            'zigbee2mqtt/bulb_color/availability',
-          'online',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
+            'online', {retain: true, qos: 0}, expect.any(Function));
+        await advancedTime(utils.hours(12));
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb_new_name/availability',
+            'offline', {retain: true, qos: 0}, expect.any(Function));
     });
 
-    it('Should retrieve the state when device is turned on/off within availability timeout', async () => {
-        MQTT.publish.mockClear();
-        getExtension().state = {};
-        const payload = {device: zigbeeHerdsman.devices.bulb_color};
-        await zigbeeHerdsman.events.deviceJoined(payload);
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
-          'online',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
+    it('Deprecated - should allow to block via advanced.availability_blocklist', async () => {
+        settings.set(['advanced', 'availability_blocklist'], [devices.bulb_color.ieeeAddr]);
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
     });
 
-    it('Should retrieve the state when device is turned on/off within availability timeout on deviceAnnounce', async () => {
-        MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
-        endpoint.read.mockClear();
-        await zigbeeHerdsman.events.deviceAnnounce({device});
-        await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(3);
-        expect(endpoint.read).toHaveBeenCalledWith('genLevelCtrl', ['currentLevel']);
-        expect(endpoint.read).toHaveBeenCalledWith('genOnOff', ['onOff']);
-        expect(endpoint.read).toHaveBeenCalledWith('lightingColorCtrl', ['currentX', 'currentY', 'colorTemperature']);
+    it('Deprecated - should allow to pass certain devices via availability_passlist', async () => {
+        settings.set(['advanced', 'availability_passlist'], [devices.bulb_color_2.ieeeAddr]);
+        settings.changeEntityOptions(devices.bulb_color_2.ieeeAddr, {availability: null});
+        await resetExtension();
+
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(0);
+        expect(devices.bulb_color_2.ping).toHaveBeenCalledTimes(1);
     });
 
-    it('Should not retrieve the state when device is turned on/off within availability timeout on deviceJoined', async () => {
-        MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const endpoint = device.getEndpoint(1);
-        endpoint.read.mockClear();
-        await zigbeeHerdsman.events.deviceJoined({device});
-        await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(0);
-    });
+    it('Deprecated - should allow to enable via availability_timeout', async () => {
+        settings.set(['availability'], false);
+        settings.set(['advanced', 'availability_timeout'], 60);
+        await resetExtension();
 
-    it('Should not do anything when message has no device', async () => {
-        MQTT.publish.mockClear();
-        const device = zigbeeHerdsman.devices.bulb_color;
-        const payload = {ieeeAddr: device.ieeeAddr};
-        await zigbeeHerdsman.events.deviceLeave(payload);
-        await flushPromises();
-        expect(MQTT.publish.mock.calls.find((c) => c[0].includes('availability'))).toBeUndefined();
-    });
-
-    it('Should not ping devices on blocklist by friendly name', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blocklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(0);
-    });
-
-    it('Should not ping devices on blacklist by friendly name', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blacklist'], ['bulb_color'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(0);
-    });
-
-    it('Should not ping devices on blocklist by IEEE address', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_blocklist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(0);
-    });
-
-    it('Should ping forced pingable devices', async () => {
-        const device = zigbeeHerdsman.devices.E11_G13;
-        const endpoint = device.getEndpoint(1);
-        const data = {modelID: 'test'}
-        const payload = {data, cluster: 'genOnOff', device, endpoint, type: 'readResponse', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should ping devices on passlist by friendly name if availability_passlist is set', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_passlist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should ping devices on whitelist by friendly name if availability_whitelist is set', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_whitelist'], ['bulb_color']);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should ping devices on passlist by IEEE address if availability_passlist is set', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        settings.set(['advanced', 'availability_passlist'], [device.ieeeAddr]);
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        device.ping.mockClear();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should not ping devices not in passlist if availability_passlist is set', async () => {
-        const device = zigbeeHerdsman.devices.bulb;
-        getExtension().state[device.ieeeAddr] = false;
-        settings.set(['advanced', 'availability_passlist'], ['0x000b57fffec6a5b3'])
-        await controller.stop();
-        await flushPromises();
-        controller = new Controller();
-        await controller.start();
-        await flushPromises();
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(0);
-        MQTT.publish.mockClear();
-        await zigbeeHerdsman.events.deviceAnnounce({device});
-    });
-
-    it('Should not read when device has no modelID and reconnects', async () => {
-        const device = zigbeeHerdsman.devices.nomodel;
-        getExtension().state[device.ieeeAddr] = true;
-        const endpoint = device.getEndpoint(1);
-        await zigbeeHerdsman.events.deviceAnnounce({device});
-        await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(0);
-    });
-
-    it('Should not read when device has is unsupported', async () => {
-        const device = zigbeeHerdsman.devices.unsupported_router;
-        getExtension().state[device.ieeeAddr] = true;
-        const endpoint = device.getEndpoint(1);
-        await zigbeeHerdsman.events.deviceAnnounce({device});
-        await flushPromises();
-        expect(endpoint.read).toHaveBeenCalledTimes(0);
-    });
-
-    it('Should stop pinging device when removed', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        device.ping.mockClear();
-        const endpoint = device.getEndpoint(1);
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-        settings.removeDevice(device.ieeeAddr);
-        jest.advanceTimersByTime(11 * 1000);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(1);
-    });
-
-    it('Should publish availability when end device joins', async () => {
-        const device = zigbeeHerdsman.devices.WXKG02LM;
-        const payload = {device};
-        MQTT.publish.mockClear();
-        await zigbeeHerdsman.events.deviceJoined(payload);
-        await flushPromises();
-        expect(device.ping).toHaveBeenCalledTimes(0);
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/button_double_key/availability',
-          'online',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
-    });
-
-    it('Should mark non-pingable device as non-available when offline for longer than 24 hours', async () => {
-        const device = zigbeeHerdsman.devices.remote;
-        const defaultLastSeen = device.lastSeen;
-        device.lastSeen = Date.now();
-        MQTT.publish.mockClear();
-        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
-        device.lastSeen = device.lastSeen - (1000 * 60 * 60 * 25);
-        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/remote/availability',
-          'offline',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
-
-        // Shouldn't do anything more when device is removed
-        settings.removeDevice(device.ieeeAddr);
-        jest.advanceTimersByTime(1000 * 60 * 60 * 1); // 1 hours
-        expect(MQTT.publish).toHaveBeenCalledTimes(1);
-
-        device.lastSeen = defaultLastSeen;
-    });
-
-    it('Should republish existing state on MQTT connected', async () => {
-        const device = zigbeeHerdsman.devices.bulb_color;
-        await controller.stop();
-        await flushPromises();
-        MQTT.publish.mockClear();
-        controller = new Controller();
-        getExtension().state[device.ieeeAddr] = false;
-        await controller.start();
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
-          'offline',
-          { retain: true, qos: 0 },
-          expect.any(Function)
-        );
-    });
-
-    it('Should clear retained availability topic when device is remove', async () => {
-        MQTT.publish.mockClear();
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/remove', 'bulb_color');
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith(
-            'zigbee2mqtt/bulb_color/availability',
-            null,
-            {retain: true, qos: 0}, expect.any(Function)
-        );
+        await advancedTime(utils.minutes(12));
+        expect(devices.bulb_color.ping).toHaveBeenCalledTimes(1);
     });
 });

test/bind.test.js

@@ -4,13 +4,15 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const stringify = require('json-stable-stringify-without-jsonify');
+jest.mock('debounce', () => jest.fn(fn => fn));
+const debounce = require('debounce');
 
 describe('Bind', () => {
     let controller;
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -21,39 +23,63 @@ describe('Bind', () => {
         }
     }
 
-    beforeEach(async () => {
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        settings.set(['experimental', 'new_api'], true);
-        controller = new Controller();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Bind');
+        await controller.enableDisableExtension(true, 'Bind');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+    });
+
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        zigbeeHerdsman.groups.group_1.members = [];
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).configureReporting.mockClear();
+        zigbeeHerdsman.devices.bulb_color.getEndpoint(1).bind.mockClear();
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        debounce.mockClear();
+        await resetExtension();
         MQTT.publish.mockClear();
     });
 
-    it('Should subscribe to topics', async () => {
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/bind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/request/device/unbind');
-        expect(MQTT.subscribe).toHaveBeenCalledWith('zigbee2mqtt/bridge/bind/#');
-    });
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
 
-    it('Should bind', async () => {
+    it('Should bind to device and configure reporting', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+
+        // Setup
+        const originalDeviceOutputClusters = device.getEndpoint(1).outputClusters;
+        device.getEndpoint(1).outputClusters = [...device.getEndpoint(1).outputClusters, 768];
+        const originalTargetBinds = target.binds;
+        target.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target.getClusterAttributeValue.mockImplementationOnce((cluster, value) =>  undefined);
         mockClear(device);
-        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'bulb_color'}));
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({transaction: "1234", from: 'remote', to: 'bulb_color'}));
         await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(3);
+        expect(target.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorCapabilities' ]);
+        expect(endpoint.bind).toHaveBeenCalledTimes(4);
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(5);
+        expect(endpoint.bind).toHaveBeenCalledWith("lightingColorCtrl", target);
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":3600,"reportableChange":1}]);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
-            stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            stringify({"transaction": "1234","data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl", "lightingColorCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
 
@@ -63,6 +89,10 @@ describe('Bind', () => {
           { retain: true, qos: 0 },
           expect.any(Function)
         );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).outputClusters = originalDeviceOutputClusters;
     });
 
     it('Should bind only specifief clusters', async () => {
@@ -99,19 +129,44 @@ describe('Bind', () => {
     it('Should unbind', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.bulb_color.getEndpoint(1);
+
+        // setup
+        target.configureReporting.mockImplementationOnce(() => {throw new Error("timeout")});
+        const originalRemoteBinds = device.getEndpoint(1).binds;
+        device.getEndpoint(1).binds = [];
+        const originalTargetBinds = target.binds;
+        target.binds = [
+            {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)},
+            {cluster: {name: 'lightingColorCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}
+        ];
+
         const endpoint = device.getEndpoint(1);
         mockClear(device);
+        delete zigbeeHerdsman.devices.bulb_color.meta.configured;
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(undefined);
         MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'bulb_color'}));
         await flushPromises();
         expect(endpoint.unbind).toHaveBeenCalledTimes(3);
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+
+        // Disable reporting
+        expect(target.configureReporting).toHaveBeenCalledTimes(3);
+        expect(target.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 0xFFFF, "minimumReportInterval": 5, "reportableChange": 1}]);
+        expect(target.configureReporting).toHaveBeenCalledWith("lightingColorCtrl",[{"attribute":"colorTemperature","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentX","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1},{"attribute":"currentY","minimumReportInterval":5,"maximumReportInterval":0xFFFF,"reportableChange":1}]);
+        expect(zigbeeHerdsman.devices.bulb_color.meta.configured).toBe(2);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/unbind',
             stringify({"data":{"from":"remote","to":"bulb_color","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        // Teardown
+        target.binds = originalTargetBinds;
+        device.getEndpoint(1).binds = originalRemoteBinds;
     });
 
     it('Should unbind coordinator', async () => {
@@ -136,7 +191,10 @@ describe('Bind', () => {
     it('Should bind to groups', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
         const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
         mockClear(device);
         MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'remote', to: 'group_1'}));
         await flushPromises();
@@ -144,11 +202,89 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/bind',
             stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
             {retain: false, qos: 0}, expect.any(Function)
         );
+
+        // Should configure reproting for device added to group
+        target1Member.configureReporting.mockClear();
+        await MQTT.events.message('zigbee2mqtt/bridge/group/group_1/add', 'bulb');
+        await flushPromises();
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genOnOff",[{"attribute": "onOff", "maximumReportInterval": 3600, "minimumReportInterval": 0, "reportableChange": 0}]);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith("genLevelCtrl",[{"attribute": "currentLevel", "maximumReportInterval": 3600, "minimumReportInterval": 5, "reportableChange": 1}]);
+    });
+
+    it('Should unbind from group', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1'}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
+        expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/unbind',
+            stringify({"data":{"from":"remote","to":"group_1","clusters":["genScenes","genOnOff","genLevelCtrl"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
+    it('Should unbind from group with skip_disable_reporting=true', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: true}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set to false, we don't expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(0);
+        endpoint.binds = originalBinds;
+    });
+
+    it('Should unbind from group with skip_disable_reporting=false', async () => {
+        const device = zigbeeHerdsman.devices.remote;
+        const target = zigbeeHerdsman.groups.group_1;
+        const target1Member = zigbeeHerdsman.devices.bulb_2.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        target.members.push(target1Member);
+
+        // The device unbind mock doesn't remove binds, therefore remove them here already otherwise configure reporiting is not disabled.
+        const originalBinds = endpoint.binds;
+        endpoint.binds = [];
+
+        target1Member.binds = [{cluster: {name: 'genLevelCtrl'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}, {cluster: {name: 'genOnOff'}, target: zigbeeHerdsman.devices.coordinator.getEndpoint(1)}];
+        target1Member.configureReporting.mockClear();
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/unbind', stringify({from: 'remote', to: 'group_1', skip_disable_reporting: false}));
+        await flushPromises();
+        expect(endpoint.unbind).toHaveBeenCalledTimes(3);
+        // with skip_disable_reporting set, we expect it to reconfigure reporting
+        expect(target1Member.configureReporting).toHaveBeenCalledTimes(2);
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genLevelCtrl', [{"attribute": "currentLevel", "maximumReportInterval": 65535, "minimumReportInterval": 5, "reportableChange": 1}])
+        expect(target1Member.configureReporting).toHaveBeenCalledWith('genOnOff', [{"attribute": "onOff", "maximumReportInterval": 65535, "minimumReportInterval": 0, "reportableChange": 0}])
+        endpoint.binds = originalBinds;
     });
 
     it('Should bind to group by number', async () => {
@@ -201,6 +337,22 @@ describe('Bind', () => {
         );
     });
 
+    it('Should bind server clusters to client clusters', async () => {
+        const device = zigbeeHerdsman.devices.temperature_sensor;
+        const target = zigbeeHerdsman.devices.heating_actuator.getEndpoint(1);
+        const endpoint = device.getEndpoint(1);
+        mockClear(device);
+        MQTT.events.message('zigbee2mqtt/bridge/request/device/bind', stringify({from: 'temperature_sensor', to: 'heating_actuator'}));
+        await flushPromises();
+        expect(endpoint.bind).toHaveBeenCalledTimes(1);
+        expect(endpoint.bind).toHaveBeenCalledWith("msTemperatureMeasurement", target);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/bridge/response/device/bind',
+            stringify({"data":{"from":"temperature_sensor","to":"heating_actuator","clusters":["msTemperatureMeasurement"],"failed":[]},"status":"ok"}),
+            {retain: false, qos: 0}, expect.any(Function)
+        );
+    });
+
     it('Should bind to default endpoint returned by endpoints()', async () => {
         const device = zigbeeHerdsman.devices.remote;
         const target = zigbeeHerdsman.devices.QBKG04LM.getEndpoint(2);
@@ -274,7 +426,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -305,7 +456,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'bulb_color', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -326,7 +476,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'Coordinator', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -346,7 +495,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -366,7 +514,6 @@ describe('Bind', () => {
         expect(endpoint.bind).toHaveBeenCalledWith("genOnOff", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genLevelCtrl", target);
         expect(endpoint.bind).toHaveBeenCalledWith("genScenes", target);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_bind', message: {from: 'remote', to: 'group_1', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -420,7 +567,6 @@ describe('Bind', () => {
         expect(endpoint.unbind).toHaveBeenCalledWith("genOnOff", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genLevelCtrl", 901);
         expect(endpoint.unbind).toHaveBeenCalledWith("genScenes", 901);
-        expect(MQTT.publish).toHaveBeenCalledTimes(4);
         expect(MQTT.publish.mock.calls[0][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[0][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genScenes'}});
         expect(MQTT.publish.mock.calls[1][0]).toStrictEqual('zigbee2mqtt/bridge/log');
@@ -428,4 +574,50 @@ describe('Bind', () => {
         expect(MQTT.publish.mock.calls[2][0]).toStrictEqual('zigbee2mqtt/bridge/log');
         expect(JSON.parse(MQTT.publish.mock.calls[2][1])).toStrictEqual({type: 'device_unbind', message: {from: 'remote', to: 'default_bind_group', cluster: 'genLevelCtrl'}});
     });
+
+    it('Should poll bounded Hue bulb when receiving message from Hue dimmer', async () => {
+        const remote = zigbeeHerdsman.devices.remote;
+        const data = {"button":3,"unknown1":3145728,"type":2,"unknown2":0,"time":1};
+        const payload = {data, cluster: 'manuSpecificPhilips', device: remote, endpoint: remote.getEndpoint(2), type: 'commandHueNotification', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(1);
+        expect(zigbeeHerdsman.devices.bulb_color.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+    });
+
+    it('Should poll bounded Hue bulb when receiving message from scene controller', async () => {
+        const remote = zigbeeHerdsman.devices.bj_scene_switch;
+        const data = {"action": "recall_2_row_1"};
+        const payload = {data, cluster: 'genScenes', device: remote, endpoint: remote.getEndpoint(10), type: 'commandRecall', linkquality: 10, groupID: 0};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        // Calls to three clusters are expected in this case
+        expect(debounce).toHaveBeenCalledTimes(3);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+	expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("lightingColorCtrl", ["currentX", "currentY", "colorTemperature"]);
+    });
+
+    it('Should poll grouped Hue bulb when receiving message from TRADFRI remote', async () => {
+        zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read.mockClear();
+        zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read.mockClear();
+        const remote = zigbeeHerdsman.devices.tradfri_remote;
+        const data = {"stepmode":0,"stepsize":43,"transtime":5};
+        const payload = {data, cluster: 'genLevelCtrl', device: remote, endpoint: remote.getEndpoint(1), type: 'commandStepWithOnOff', linkquality: 10, groupID: 15071};
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genLevelCtrl", ["currentLevel"]);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledWith("genOnOff", ["onOff"]);
+
+        // Should also only debounce once
+        await zigbeeHerdsman.events.message(payload);
+        await flushPromises();
+        expect(debounce).toHaveBeenCalledTimes(2);
+        expect(zigbeeHerdsman.devices.bulb_color_2.getEndpoint(1).read).toHaveBeenCalledTimes(4);
+
+        // Should only call Hue bulb, not e.g. tradfri
+        expect(zigbeeHerdsman.devices.bulb_2.getEndpoint(1).read).toHaveBeenCalledTimes(0);
+    });
 });

test/configure.test.js

@@ -4,7 +4,7 @@ const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 const stringify = require('json-stable-stringify-without-jsonify');
 
@@ -12,13 +12,14 @@ const mocksClear = [MQTT.publish, logger.warn, logger.debug];
 
 describe('Configure', () => {
     let controller;
+    let coordinatorEndpoint;
 
-    expectRemoteConfigured = () => {
+    const expectRemoteConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(2);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', this.coordinatorEndoint);
-        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', this.coordinatorEndoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genOnOff', coordinatorEndpoint);
+        expect(endpoint1.bind).toHaveBeenCalledWith('genLevelCtrl', coordinatorEndpoint);
 
         const endpoint2 = device.getEndpoint(2);
         expect(endpoint2.write).toHaveBeenCalledTimes(1);
@@ -26,13 +27,27 @@ describe('Configure', () => {
         expect(device.meta.configured).toBe(1);
     }
 
-    expectRemoteNotConfigured = () => {
+    const expectBulbConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(2);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', ['colorCapabilities']);
+        expect(endpoint1.read).toHaveBeenCalledWith('lightingColorCtrl', [ 'colorTempPhysicalMin', 'colorTempPhysicalMax' ]);
+    }
+
+    const expectBulbNotConfigured = () => {
+        const device = zigbeeHerdsman.devices.bulb;
+        const endpoint1 = device.getEndpoint(1);
+        expect(endpoint1.read).toHaveBeenCalledTimes(0);
+    }
+
+    const expectRemoteNotConfigured = () => {
         const device = zigbeeHerdsman.devices.remote;
         const endpoint1 = device.getEndpoint(1);
         expect(endpoint1.bind).toHaveBeenCalledTimes(0);
     }
 
-    mockClear = (device) => {
+    const mockClear = (device) => {
         for (const endpoint of device.endpoints) {
             endpoint.read.mockClear();
             endpoint.write.mockClear();
@@ -41,68 +56,79 @@ describe('Configure', () => {
         }
     }
 
-    beforeEach(async () => {
-        jest.useRealTimers();
-        data.writeDefaultConfiguration();
-        settings._reRead();
-        data.writeEmptyState();
-        controller = new Controller();
+    let resetExtension = async () => {
+        await controller.enableDisableExtension(false, 'Configure');
+        await controller.enableDisableExtension(true, 'Configure');
+    }
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+        controller = new Controller(jest.fn(), jest.fn());
         await controller.start();
-        mocksClear.forEach((m) => m.mockClear());
         await flushPromises();
-        this.coordinatorEndoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
     });
 
-    it('Should configure on startup', async () => {
-        expectRemoteConfigured();
+    beforeEach(async () => {
+        data.writeDefaultConfiguration();
+        settings.reRead();
+        mocksClear.forEach((m) => m.mockClear());
+     coordinatorEndpoint = zigbeeHerdsman.devices.coordinator.getEndpoint(1);
+        await resetExtension();
+    });
+
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
+    it('Should configure Router on startup', async () => {
+        expectBulbConfigured();
+    });
+
+    it('Should not configure EndDevice on startup', async () => {
+        expectRemoteNotConfigured();
     });
 
     it('Should re-configure when device rejoins', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         const endpoint = device.getEndpoint(1);
         await flushPromises();
         mockClear(device);
         const payload = {device};
         zigbeeHerdsman.events.deviceJoined(payload);
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should reconfigure reporting on reportingDisabled event', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
-        mockClear(device);
-        expectRemoteNotConfigured();
-        controller.eventBus.emit('reportingDisabled', {device})
+        expectBulbConfigured();
+        const device = controller.zigbee.resolveEntity(zigbeeHerdsman.devices.bulb);
+        mockClear(device.zh);
+        expectBulbNotConfigured();
+        controller.eventBus.emitReportingDisabled({device});
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should not configure twice', async () => {
-        expectRemoteConfigured();
-        const device = zigbeeHerdsman.devices.remote;
-        const endpoint = device.getEndpoint(1);
+        expectBulbConfigured();
+        const device = zigbeeHerdsman.devices.bulb;
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.deviceInterview({device});
         await flushPromises();
-        expect(endpoint.bind).toHaveBeenCalledTimes(0);
+        expectBulbNotConfigured();
     });
 
     it('Should configure on zigbee message when not configured yet', async () => {
-        const device = zigbeeHerdsman.devices.remote;
+        const device = zigbeeHerdsman.devices.bulb;
         delete device.meta.configured;
-        const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
-        expectRemoteConfigured();
+        expectBulbConfigured();
     });
 
     it('Should allow to configure via MQTT', async () => {
-        expectRemoteConfigured();
         mockClear(zigbeeHerdsman.devices.remote);
         expectRemoteNotConfigured();
         await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', 'remote');
@@ -137,11 +163,11 @@ describe('Configure', () => {
     });
 
     it('Fail to configure via MQTT when device has no configure', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "bulb", transaction: 20}));
+        await MQTT.events.message('zigbee2mqtt/bridge/request/device/configure', stringify({id: "0x90fd9ffffe4b64ax", transaction: 20}));
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith(
             'zigbee2mqtt/bridge/response/device/configure',
-            stringify({"data":{"id": "bulb"},"status":"error","error": "Device 'bulb' cannot be configured","transaction":20}),
+            stringify({"data":{"id": "0x90fd9ffffe4b64ax"},"status":"error","error": "Device 'ZNLDP12LM' cannot be configured","transaction":20}),
             {retain: false, qos: 0}, expect.any(Function)
         );
     });
@@ -161,9 +187,9 @@ describe('Configure', () => {
     });
 
     it('Legacy api: Should skip reconfigure when device does not require this', async () => {
-        await MQTT.events.message('zigbee2mqtt/bridge/configure', 'bulb');
+        await MQTT.events.message('zigbee2mqtt/bridge/configure', '0x90fd9ffffe4b64ax');
         await flushPromises();
-        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'bulb', device does not require this.`)
+        expect(logger.warn).toHaveBeenCalledWith(`Skipping configure of 'ZNLDP12LM', device does not require this.`)
     });
 
     it('Should not configure when interviewing', async () => {
@@ -172,8 +198,7 @@ describe('Configure', () => {
         device.interviewing = true;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectRemoteNotConfigured();
         device.interviewing = false;
@@ -185,8 +210,7 @@ describe('Configure', () => {
         device.interviewCompleted = false;
         const endpoint = device.getEndpoint(1);
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expectRemoteConfigured();
         device.interviewCompleted = true;
@@ -198,11 +222,10 @@ describe('Configure', () => {
         const endpoint = device.getEndpoint(1);
         endpoint.bind.mockImplementationOnce(async () => await wait(500));
         mockClear(device);
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(1);
     });
@@ -214,20 +237,19 @@ describe('Configure', () => {
         const endpoint = device.getEndpoint(1);
         mockClear(device);
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        const payload = {data: {zclVersion: 1}, cluster: 'genBasic', device, endpoint, type: 'attributeReport', linkquality: 10};
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         endpoint.bind.mockImplementationOnce(async () => {throw new Error('BLA')});
-        await zigbeeHerdsman.events.message(payload);
+        await zigbeeHerdsman.events.lastSeenChanged({device});
         await flushPromises();
         expect(endpoint.bind).toHaveBeenCalledTimes(3);
     });

test/controller.test.js

@@ -3,14 +3,13 @@ const logger = require('./stub/logger');
 const zigbeeHerdsman = require('./stub/zigbeeHerdsman');
 const MQTT = require('./stub/mqtt');
 const path = require('path');
-const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {});
 const settings = require('../lib/util/settings');
 const Controller = require('../lib/controller');
 const stringify = require('json-stable-stringify-without-jsonify');
-const flushPromises = () => new Promise(setImmediate);
+const flushPromises = require('./lib/flushPromises');
 const tmp = require('tmp');
 const mocksClear = [
-    zigbeeHerdsman.permitJoin, mockExit, MQTT.end, zigbeeHerdsman.stop, logger.debug,
+    zigbeeHerdsman.permitJoin, MQTT.end, zigbeeHerdsman.stop, logger.debug,
     MQTT.publish, MQTT.connect, zigbeeHerdsman.devices.bulb_color.removeFromNetwork,
     zigbeeHerdsman.devices.bulb.removeFromNetwork, logger.error,
 ];
@@ -19,30 +18,41 @@ const fs = require('fs');
 
 describe('Controller', () => {
     let controller;
+    let mockExit;
+
+    beforeAll(async () => {
+        jest.useFakeTimers();
+    });
 
     beforeEach(() => {
+        MQTT.restoreOnMock();
         zigbeeHerdsman.returnDevices.splice(0);
-        controller = new Controller();
+        mockExit = jest.fn();
+        controller = new Controller(jest.fn(), mockExit);
         mocksClear.forEach((m) => m.mockClear());
         data.writeDefaultConfiguration();
-        settings._reRead();
+        settings.reRead();
         data.writeDefaultState();
     });
 
+    afterAll(async () => {
+        jest.useRealTimers();
+    })
+
     it('Start controller', async () => {
         await controller.start();
-        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}});
+        expect(zigbeeHerdsman.constructor).toHaveBeenCalledWith({"network":{"panID":6754,"extendedPanID":[221,221,221,221,221,221,221,221],"channelList":[11],"networkKey":[1,3,5,7,9,11,13,15,0,2,4,6,8,10,12,13]},"databasePath":path.join(data.mockDir, "database.db"), "databaseBackupPath":path.join(data.mockDir, "database.db.backup"),"backupPath":path.join(data.mockDir, "coordinator_backup.json"),"acceptJoiningDeviceHandler": expect.any(Function),adapter: {concurrent: null, delay: null, disableLED: false}, "serialPort":{"baudRate":undefined,"rtscts":undefined,"path":"/dev/dummy"}}, logger);
         expect(zigbeeHerdsman.start).toHaveBeenCalledTimes(1);
         expect(zigbeeHerdsman.setLED).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.setTransmitPower).toHaveBeenCalledTimes(0);
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(true, undefined, undefined);
         expect(logger.info).toHaveBeenCalledWith(`Currently ${Object.values(zigbeeHerdsman.devices).length - 1} devices are joined:`)
         expect(logger.info).toHaveBeenCalledWith('bulb (0x000b57fffec6a5b2): LED1545G12 - IKEA TRADFRI LED bulb E26/E27 980 lumen, dimmable, white spectrum, opal white (Router)');
         expect(logger.info).toHaveBeenCalledWith('remote (0x0017880104e45517): 324131092621 - Philips Hue dimmer switch (EndDevice)');
         expect(logger.info).toHaveBeenCalledWith('0x0017880104e45518 (0x0017880104e45518): Not supported (EndDevice)');
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
-        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"}});
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", {"will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state", "qos": 1}});
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}),{ retain: true, qos: 0 }, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/remote', stringify({"brightness":255}), { retain: true, qos: 0 }, expect.any(Function));
     });
@@ -78,7 +88,7 @@ describe('Controller', () => {
         await flushPromises();
         expect(MQTT.connect).toHaveBeenCalledTimes(1);
         const expected = {
-            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state"},
+            "will": {"payload": "offline", "retain": true, "topic": "zigbee2mqtt/bridge/state", "qos": 1},
             keepalive: 30,
             ca: Buffer.from([99, 97]),
             key: Buffer.from([107, 101, 121]),
@@ -93,12 +103,15 @@ describe('Controller', () => {
         expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
-    it('Should generate network_key when set to GENERATE', async () => {
+    it('Should generate network_key and pan_id when set to GENERATE', async () => {
         settings.set(['advanced', 'network_key'], 'GENERATE');
+        settings.set(['advanced', 'pan_id'], 'GENERATE');
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.networkKey.length).toStrictEqual(16);
+        expect(zigbeeHerdsman.constructor.mock.calls[0][0].network.panID).toStrictEqual(expect.any(Number));
         expect(data.read().advanced.network_key.length).toStrictEqual(16);
+        expect(data.read().advanced.pan_id).toStrictEqual(expect.any(Number));
     });
 
     it('Start controller should publish cached states', async () => {
@@ -128,14 +141,12 @@ describe('Controller', () => {
     });
 
     it('Log when MQTT client is unavailable', async () => {
-        jest.useFakeTimers();
         await controller.start();
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
         jest.advanceTimersByTime(11 * 1000);
         await flushPromises();
-        expect(logger.error).toHaveBeenCalledTimes(1);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
         controller.mqtt.client.reconnecting = false;
     });
@@ -145,7 +156,8 @@ describe('Controller', () => {
         await flushPromises();
         logger.error.mockClear();
         controller.mqtt.client.reconnecting = true;
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON', brightness: 50, color_temp: 370, color: {r: 100, g: 50, b: 10}, dummy: {1: 'yes', 2: 'no'}});
         await flushPromises();
         expect(logger.error).toHaveBeenCalledTimes(2);
         expect(logger.error).toHaveBeenCalledWith("Not connected to MQTT server!");
@@ -162,6 +174,7 @@ describe('Controller', () => {
 
     it('Should remove device not on passlist on startup', async () => {
         settings.set(['passlist'], [zigbeeHerdsman.devices.bulb_color.ieeeAddr]);
+        zigbeeHerdsman.devices.bulb.removeFromNetwork.mockImplementationOnce(() => {throw new Error("dummy")});
         await controller.start();
         await flushPromises();
         expect(zigbeeHerdsman.devices.bulb_color.removeFromNetwork).toHaveBeenCalledTimes(0);
@@ -198,11 +211,22 @@ describe('Controller', () => {
         expect(mockExit).toHaveBeenCalledTimes(1);
     });
 
+    it('Start controller fails due to MQTT', async () => {
+        MQTT.on.mockImplementation((type, handler) => {
+            if (type === 'error') handler({message: 'addr not found'});
+        });
+        await controller.start();
+        await flushPromises();
+        expect(logger.error).toHaveBeenCalledWith('MQTT failed to connect: addr not found');
+        expect(mockExit).toHaveBeenCalledTimes(1);
+        expect(mockExit).toHaveBeenCalledWith(1);
+    });
+
     it('Start controller with permit join true', async () => {
         settings.set(['permit_join'], false);
         await controller.start();
         expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledTimes(1);
-        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false);
+        expect(zigbeeHerdsman.permitJoin).toHaveBeenCalledWith(false, undefined, undefined);
     });
 
     it('Start controller with disable_led', async () => {
@@ -256,6 +280,17 @@ describe('Controller', () => {
         expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'dummytopic' with data 'dummymessage'")
     });
 
+    it('Skip MQTT messages on topic we published to', async () => {
+        await controller.start();
+        logger.debug.mockClear();
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledWith("Received MQTT message on 'zigbee2mqtt/skip-this-topic' with data 'skipped'")
+        logger.debug.mockClear();
+        await controller.mqtt.publish('skip-this-topic', '', {});
+        await MQTT.events.message('zigbee2mqtt/skip-this-topic', 'skipped');
+        expect(logger.debug).toHaveBeenCalledTimes(0);
+    });
+
     it('On zigbee event message', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
@@ -280,17 +315,6 @@ describe('Controller', () => {
         expect(settings.getDevice(device.ieeeAddr)).not.toBeNull();
     });
 
-    it('On zigbee event message from unkown device should create it', async () => {
-        await controller.start();
-        const device = zigbeeHerdsman.devices.notInSettings;
-        settings.removeDevice(device.ieeeAddr);
-        expect(settings.getDevice(device.ieeeAddr)).toBeNull();
-        const payload = {device, endpoint: device.getEndpoint(1), type: 'attributeReport', linkquality: 10, groupID: 0, cluster: 'genBasic', data: {modelId: device.modelID}};
-        await zigbeeHerdsman.events.message(payload);
-        await flushPromises();
-        expect(settings.getDevice(device.ieeeAddr)).toStrictEqual({"ID": "0x0017880104e45519", "friendlyName": "0x0017880104e45519", "friendly_name": "0x0017880104e45519"});
-    });
-
     it('On zigbee deviceJoined', async () => {
         await controller.start();
         const device = zigbeeHerdsman.devices.bulb;
@@ -431,7 +455,8 @@ describe('Controller', () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute');
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {dummy: {1: 'yes', 2: 'no'}, color: {r: 100, g: 50, b: 10}, state: 'ON', test: undefined, test1: null, color_temp: 370, brightness: 50});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {dummy: {1: 'yes', 2: 'no'}, color: {r: 100, g: 50, b: 10}, state: 'ON', test: undefined, test1: null, color_temp: 370, brightness: 50});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/brightness", "50", {"qos": 0, "retain": true}, expect.any(Function));
@@ -447,7 +472,8 @@ describe('Controller', () => {
         await controller.start();
         settings.set(['experimental', 'output'], 'attribute_and_json');
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(5);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
@@ -463,7 +489,8 @@ describe('Controller', () => {
         settings.set(['experimental', 'output'], 'attribute_and_json');
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'filtered_attributes'], ['color_temp', 'linkquality']);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
@@ -476,7 +503,8 @@ describe('Controller', () => {
         settings.set(['experimental', 'output'], 'attribute_and_json');
         settings.set(['device_options', 'filtered_attributes'], ['color_temp', 'linkquality']);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON', brightness: 200, color_temp: 370, linkquality: 99});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(3);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb/state", "ON", {"qos": 0, "retain": true}, expect.any(Function));
@@ -488,12 +516,14 @@ describe('Controller', () => {
         await controller.start();
         settings.set(['mqtt', 'include_device_information'], true);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
+        let device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99,"device":{"friendlyName":"bulb","model":"LED1545G12","ieeeAddr":"0x000b57fffec6a5b2","networkAddress":40369,"type":"Router","manufacturerID":4476,"powerSource":"Mains (single phase)","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": true}, expect.any(Function));
 
         // Unsupported device should have model "unknown"
-        await controller.publishEntityState('unsupported2', {state: 'ON'});
+        device = controller.zigbee.resolveEntity('unsupported2');
+        await controller.publishEntityState(device, {state: 'ON'});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/unsupported2', stringify({"state":"ON","device":{"friendlyName":"unsupported2","model":"unknown","ieeeAddr":"0x0017880104e45529","networkAddress":6536,"type":"EndDevice","manufacturerID":0,"powerSource":"Battery","dateCode":null, "softwareBuildID": null}}), {"qos": 0, "retain": false}, expect.any(Function));
     });
@@ -502,7 +532,8 @@ describe('Controller', () => {
         await controller.start();
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], false);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": false}, expect.any(Function));
     });
@@ -511,7 +542,8 @@ describe('Controller', () => {
         await controller.start();
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true}, expect.any(Function));
     });
@@ -522,7 +554,8 @@ describe('Controller', () => {
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retain'], true);
         settings.set(['devices', zigbeeHerdsman.devices.bulb.ieeeAddr, 'retention'], 37);
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/bulb', stringify({"state":"ON","brightness":50,"color_temp":370,"linkquality":99}), {"qos": 0, "retain": true, "properties": {messageExpiryInterval: 37}}, expect.any(Function));
     });
@@ -531,7 +564,8 @@ describe('Controller', () => {
         data.writeEmptyState();
         await controller.start();
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(0);
     });
@@ -541,8 +575,9 @@ describe('Controller', () => {
         data.removeState();
         await controller.start();
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
-        await controller.publishEntityState('bulb', {brightness: 200});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
+        await controller.publishEntityState(device, {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({state: "ON"}), {"qos": 0, "retain": true}, expect.any(Function));
@@ -565,22 +600,15 @@ describe('Controller', () => {
         data.writeEmptyState();
         await controller.start();
         MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb', {state: 'ON'});
-        await controller.publishEntityState('bulb', {brightness: 200});
+        const device = controller.zigbee.resolveEntity('bulb');
+        await controller.publishEntityState(device, {state: 'ON'});
+        await controller.publishEntityState(device, {brightness: 200});
         await flushPromises();
         expect(MQTT.publish).toHaveBeenCalledTimes(2);
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"state":"ON"}), {"qos": 0, "retain": true}, expect.any(Function));
         expect(MQTT.publish).toHaveBeenCalledWith("zigbee2mqtt/bulb", stringify({"brightness":200}), {"qos": 0, "retain": true}, expect.any(Function));
     });
 
-    it('Publish should not do anything for unknown entity', async () => {
-        await controller.start();
-        MQTT.publish.mockClear();
-        await controller.publishEntityState('bulb-unknown', {brightness: 200});
-        await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledTimes(0);
-    });
-
     it('Should start when state is corrupted', async () => {
         fs.writeFileSync(path.join(data.mockDir, 'state.json'), 'corrupted');
         await controller.start();
@@ -588,14 +616,45 @@ describe('Controller', () => {
         expect(controller.state.state).toStrictEqual({});
     });
 
-    it('Load user extension', async () => {
-        const extensionPath = path.join(data.mockDir, 'extension');
-        fs.mkdirSync(extensionPath);
-        fs.copyFileSync(path.join(__dirname, 'assets', 'exampleExtension.js'), path.join(extensionPath, 'exampleExtension.js'))
-        controller = new Controller();
+    it('Start controller with force_disable_retain', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
         await controller.start();
         await flushPromises();
-        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/example/extension', 'test', { retain: false, qos: 0 }, expect.any(Function));
+        expect(MQTT.connect).toHaveBeenCalledTimes(1);
+        const expected = {
+            "will": { "payload": "offline", "retain": false, "topic": "zigbee2mqtt/bridge/state", "qos": 1 },
+        }
+        expect(MQTT.connect).toHaveBeenCalledWith("mqtt://localhost", expected);
     });
 
+    it('Should prevent any message being published with retain flag when force_disable_retain is set', async () => {
+        settings.set(['mqtt', 'force_disable_retain'], true);
+        await controller.mqtt.connect()
+        MQTT.publish.mockClear();
+        await controller.mqtt.publish('fo', 'bar', { retain: true })
+        await flushPromises();
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith('zigbee2mqtt/fo', 'bar', { retain: false, qos: 0 }, expect.any(Function));
+    });
+
+    it('Should disable legacy options on new network start', async () => {
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeTruthy();
+        expect(settings.get().advanced.legacy_api).toBeTruthy();
+        zigbeeHerdsman.start.mockReturnValueOnce('reset');
+        await controller.start();
+        expect(settings.get().advanced.homeassistant_legacy_entity_attributes).toBeFalsy();
+        expect(settings.get().advanced.legacy_api).toBeFalsy();
+    });
+
+    it('Should publish last seen changes', async () => {
+        settings.set(['advanced', 'last_seen'], 'epoch');
+        await controller.start();
+        await flushPromises();
+        MQTT.publish.mockClear();
+        const device = zigbeeHerdsman.devices.remote;
+        await zigbeeHerdsman.events.lastSeenChanged({device});
+        expect(MQTT.publish).toHaveBeenCalledTimes(1);
+        expect(MQTT.publish).toHaveBeenCalledWith(
+            'zigbee2mqtt/remote', stringify({"brightness":255,"last_seen":1000}), { qos: 0, retain: true }, expect.any(Function));
+    });
 });

test/data.test.js

@@ -1,5 +1,5 @@
 const logger = require('./stub/logger');
-const data = require('../lib/util/data.js');
+const data = require('../lib/util/data').default;
 const path = require('path');
 const tmp = require('tmp');
 const fs = require('fs');
@@ -15,12 +15,12 @@ describe('Data', () => {
         it('Should return correct path when ZIGBEE2MQTT_DATA set', () => {
             const expected = tmp.dirSync().name;
             process.env.ZIGBEE2MQTT_DATA = expected;
-            data._reload();
+            data.__testingOnly_reload();
             const actual = data.getPath();
             expect(actual).toBe(expected);
             expect(data.joinPath('test')).toStrictEqual(path.join(expected, 'test'));
             delete process.env.ZIGBEE2MQTT_DATA;
-            data._reload();
+            data.__testingOnly_reload();
         });
     });
 });